[llvm] [SYCL] E2E LIT tests initial configs & test (PR #177407)
Kseniya Tikhomirova via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 23 09:03:12 PST 2026
https://github.com/KseniyaTikhomirova updated https://github.com/llvm/llvm-project/pull/177407
>From 6a8d6d8d8a2a09e6274905a5aca328702bfdacd2 Mon Sep 17 00:00:00 2001
From: "Tikhomirova, Kseniya" <kseniya.tikhomirova at intel.com>
Date: Wed, 10 Dec 2025 09:43:17 -0800
Subject: [PATCH] [SYCL] E2E LIT tests initial configs & test
Signed-off-by: Tikhomirova, Kseniya <kseniya.tikhomirova at intel.com>
---
libsycl/CMakeLists.txt | 11 +-
libsycl/test_e2e/CMakeLists.txt | 37 +++
libsycl/test_e2e/README.md | 93 ++++++++
.../test_e2e/basic/platform_get_devices.cpp | 116 +++++++++
libsycl/test_e2e/lit.cfg.py | 221 ++++++++++++++++++
libsycl/test_e2e/lit.site.cfg.py.in | 39 ++++
6 files changed, 516 insertions(+), 1 deletion(-)
create mode 100644 libsycl/test_e2e/CMakeLists.txt
create mode 100644 libsycl/test_e2e/README.md
create mode 100644 libsycl/test_e2e/basic/platform_get_devices.cpp
create mode 100644 libsycl/test_e2e/lit.cfg.py
create mode 100644 libsycl/test_e2e/lit.site.cfg.py.in
diff --git a/libsycl/CMakeLists.txt b/libsycl/CMakeLists.txt
index f25f51def0cc7..64e4b585e6d68 100644
--- a/libsycl/CMakeLists.txt
+++ b/libsycl/CMakeLists.txt
@@ -133,5 +133,14 @@ add_custom_target(libsycl-runtime-libraries
)
add_subdirectory(src)
-
add_subdirectory(tools)
+
+if(LLVM_INCLUDE_TESTS)
+ add_subdirectory(test_e2e)
+endif()
+
+add_custom_target(libsycl-toolchain ALL
+ DEPENDS libsycl-runtime-libraries
+ sycl-ls
+ COMMENT "Building SYCL compiler toolchain..."
+)
diff --git a/libsycl/test_e2e/CMakeLists.txt b/libsycl/test_e2e/CMakeLists.txt
new file mode 100644
index 0000000000000..189f6dfc21ea3
--- /dev/null
+++ b/libsycl/test_e2e/CMakeLists.txt
@@ -0,0 +1,37 @@
+cmake_minimum_required(VERSION 3.20.0)
+
+message("Configuring SYCL End-to-End Tests")
+
+set(LIBSYCL_CXX_COMPILER "${LLVM_BINARY_DIR}/bin/clang++")
+set(LIBSYCL_E2E_CXX_FLAGS "" CACHE STRING
+ "Flags passed to clang++ when building SYCL end-to-end tests")
+
+if(NOT LIBSYCL_TEST_E2E_TARGETS)
+ set(LIBSYCL_TEST_E2E_TARGETS "all")
+endif()
+
+if(NOT DEFINED LEVEL_ZERO_LIBS_DIR AND NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
+ find_path(LEVEL_ZERO_INCLUDE_DIR NAMES level_zero/ze_api.h)
+ if(LEVEL_ZERO_INCLUDE_DIR)
+ find_library(LEVEL_ZERO_LIBS_DIR NAMES ze_loader)
+ endif()
+endif()
+
+configure_lit_site_cfg(
+ ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+ ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+ MAIN_CONFIG
+ ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+
+list(APPEND LIBSYCL_E2E_TEST_DEPS
+ libsycl-toolchain
+ FileCheck
+ not
+)
+
+add_lit_testsuite(check-sycl-e2e
+ "Running SYCL End-to-End tests"
+ ${CMAKE_CURRENT_BINARY_DIR}
+ PARAMS enable_benchmarks="${_libcxx_benchmark_mode}"
+ DEPENDS ${LIBSYCL_E2E_TEST_DEPS})
diff --git a/libsycl/test_e2e/README.md b/libsycl/test_e2e/README.md
new file mode 100644
index 0000000000000..6cc6b0b649cdb
--- /dev/null
+++ b/libsycl/test_e2e/README.md
@@ -0,0 +1,93 @@
+## Getting Started
+
+This directory contains SYCL-related end-to-end tests distributed in
+subdirectories based on testing scope. `libsycl` uses LIT to configure and run
+its tests.
+
+Please see the [Lit Command Guide](https://llvm.org/docs/CommandGuide/lit.html)
+for more information about LIT.
+
+## Prerequisites
+
+* Target runtime(s) to execute tests on devices.
+ TODO: add link to liboffload instruction once they add it.
+* Compiler & libsycl. Can be built following these
+ [instructions](/libsycl/docs/index.rst).
+
+## Run the tests
+
+`libsycl` is integrated via LLVM_ENABLE_RUNTIMES and is not visible as top
+level target. Same is applicable for tests. To run `check-sycl-e2e` tests you
+need to prefix <build>/runtimes/runtimes-bins/ to the paths of all tests.
+For example, to run all the libsycl end-to-end tests you can do:
+```bash
+<build>/bin/llvm-lit <build>/runtimes/runtimes-bins/libsycl/test_e2e
+```
+
+To run individual test use the path to it instead of the top level `test_e2e`
+directory.
+
+If you used ninja as your build system, you can run all the tests in the
+libsycl testsuite as:
+
+```bash
+ ninja -C <build>/runtimes/runtimes-bins check-sycl-e2e
+ ```
+
+
+## CMake parameters
+
+These parameters can be used to configure tests:
+
+***LIBSYCL_CXX_COMPILER*** - path to compiler to use it for building tests.
+
+***LIBSYCL_E2E_CXX_FLAGS*** - flags to be passed to LIBSYCL_CXX_COMPILER when
+ building libsycl end-to-end tests.
+
+***LLVM_LIT*** - path to llvm-lit tool.
+
+***LEVEL_ZERO_INCLUDE_DIR*** - path to Level Zero headers.
+
+***LEVEL_ZERO_LIBS_DIR*** - path to Level Zero libraries.
+
+## Creating or modifying tests
+
+### LIT feature checks
+
+Following features can be checked in tests to limit test execution to the
+specific environment via REQUIRES, UNSUPPORTED, etc. filters.
+
+#### Auto-detected features
+
+The following features are automatically detected by `llvm-lit` by scanning the
+environment:
+
+* **linux** - host OS;
+* **any-device-is-gpu** - device type to be available;
+* **any-device-is-level_zero** - backend to be available;
+
+Note: sycl-ls tool doesn't have assigned feature since it is essential for tests configuration and though always available if test is executed.
+
+### llvm-lit parameters
+
+Following options can be passed to llvm-lit tool through --param option to
+configure specific single test execution in the command line:
+
+* **libsycl_compiler** - full path to compiler to use;
+* **extra_environment** - comma-separated list of variables with values to be
+ added to test environment. Can be also set by LIT_EXTRA_ENVIRONMENT variable
+ in CMake.
+* **extra_system_environment** - comma-separated list of variables to be
+ propagated from the host environment to test environment. Can be also set by
+ LIT_EXTRA_SYSTEM_ENVIRONMENT variable in CMake.
+* **level_zero_include** - directory containing Level_Zero native headers, can
+ be also set by CMake variable LEVEL_ZERO_INCLUDE_DIR.
+* **level_zero_libs_dir** - directory containing Level_Zero native libraries,
+ can be also set by CMake variable LEVEL_ZERO_LIBS_DIR.
+
+Example:
+
+```bash
+<build>/bin/llvm-lit --param libsycl_compiler=path/to/clang++ \
+ <build>/runtimes/runtimes-bins/libsycl/test_e2e
+```
\ No newline at end of file
diff --git a/libsycl/test_e2e/basic/platform_get_devices.cpp b/libsycl/test_e2e/basic/platform_get_devices.cpp
new file mode 100644
index 0000000000000..39c2f08e0993c
--- /dev/null
+++ b/libsycl/test_e2e/basic/platform_get_devices.cpp
@@ -0,0 +1,116 @@
+// RUN: %clangxx %sycl_options %s -o %t.out
+// RUN: %t.out
+//
+// Tests platform::get_devices for each device type.
+
+#include <sycl.hpp>
+
+#include <algorithm>
+#include <iostream>
+
+std::string BackendToString(sycl::backend Backend) {
+ switch (Backend) {
+ case sycl::backend::opencl:
+ return "opencl";
+ case sycl::backend::level_zero:
+ return "level_zero";
+ case sycl::backend::cuda:
+ return "cuda";
+ case sycl::backend::hip:
+ return "hip";
+ default:
+ return "unknown";
+ }
+}
+
+std::string DeviceTypeToString(sycl::info::device_type DevType) {
+ switch (DevType) {
+ case sycl::info::device_type::all:
+ return "device_type::all";
+ case sycl::info::device_type::cpu:
+ return "device_type::cpu";
+ case sycl::info::device_type::gpu:
+ return "device_type::gpu";
+ case sycl::info::device_type::accelerator:
+ return "device_type::accelerator";
+ case sycl::info::device_type::custom:
+ return "device_type::custom";
+ case sycl::info::device_type::automatic:
+ return "device_type::automatic";
+ case sycl::info::device_type::host:
+ return "device_type::host";
+ default:
+ return "UNKNOWN";
+ }
+}
+
+template <typename T1, typename T2>
+int Check(const T1 &LHS, const T2 &RHS, std::string TestName) {
+ if (LHS != RHS) {
+ std::cout << "Failed check " << LHS << " != " << RHS << ": " << TestName
+ << std::endl;
+ return 1;
+ }
+ return 0;
+}
+
+int CheckDeviceType(const sycl::platform &P, sycl::info::device_type DevType,
+ std::vector<sycl::device> &AllDevices) {
+ assert(DevType != sycl::info::device_type::all);
+ int Failures = 0;
+
+ std::vector<sycl::device> Devices = P.get_devices(DevType);
+
+ if (DevType == sycl::info::device_type::automatic) {
+ if (AllDevices.empty()) {
+ Failures += Check(
+ Devices.size(), 0,
+ "No devices reported for all query, but automatic returns a device.");
+ } else {
+ Failures += Check(Devices.size(), 1,
+ "Number of devices for device_type::automatic query.");
+ if (Devices.size())
+ Failures +=
+ Check(std::count(AllDevices.begin(), AllDevices.end(), Devices[0]),
+ 1, "Device is in the set of all devices in the platform.");
+ }
+ return Failures;
+ }
+
+ // Count devices with the type;
+ size_t DevCount = 0;
+ for (sycl::device Device : Devices)
+ DevCount += (Device.get_info<sycl::info::device::device_type>() == DevType);
+
+ Failures +=
+ Check(Devices.size(), DevCount,
+ "Unexpected number of devices for " + DeviceTypeToString(DevType));
+
+ Failures += Check(std::all_of(Devices.begin(), Devices.end(),
+ [&](const auto &Dev) {
+ return std::count(AllDevices.begin(),
+ AllDevices.end(), Dev) == 1;
+ }),
+ true,
+ "Not all devices for " + DeviceTypeToString(DevType) +
+ " appear in the list of all devices");
+
+ return Failures;
+}
+
+int main() {
+ int Failures = 0;
+ for (sycl::platform P : sycl::platform::get_platforms()) {
+ std::cout << "Checking platform with backend "
+ << BackendToString(P.get_backend()) << std::endl;
+
+ std::vector<sycl::device> Devices = P.get_devices();
+
+ for (sycl::info::device_type DevType :
+ {sycl::info::device_type::cpu, sycl::info::device_type::gpu,
+ sycl::info::device_type::accelerator, sycl::info::device_type::custom,
+ sycl::info::device_type::automatic, sycl::info::device_type::host})
+ Failures += CheckDeviceType(P, DevType, Devices);
+ }
+ return Failures;
+}
diff --git a/libsycl/test_e2e/lit.cfg.py b/libsycl/test_e2e/lit.cfg.py
new file mode 100644
index 0000000000000..fa6256507c8f0
--- /dev/null
+++ b/libsycl/test_e2e/lit.cfg.py
@@ -0,0 +1,221 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+import re
+import subprocess
+import textwrap
+import shlex
+
+from lit.llvm import llvm_config
+import lit.formats
+from lit.llvm.subst import ToolSubst, FindTool
+
+# name: The name of this test suite.
+config.name = "SYCL"
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = [".cpp"]
+
+config.excludes = ["Inputs"]
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# allow expanding substitutions that are based on other substitutions
+config.recursiveExpansionLimit = 10
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = config.libsycl_obj_root
+
+# To be filled by lit.local.cfg files.
+config.required_features = []
+config.unsupported_features = []
+
+# Cleanup environment variables which may affect tests
+possibly_dangerous_env_vars = [
+ "COMPILER_PATH",
+ "RC_DEBUG_OPTIONS",
+ "CINDEXTEST_PREAMBLE_FILE",
+ "LIBRARY_PATH",
+ "CPATH",
+ "C_INCLUDE_PATH",
+ "CPLUS_INCLUDE_PATH",
+ "OBJC_INCLUDE_PATH",
+ "OBJCPLUS_INCLUDE_PATH",
+ "LIBCLANG_TIMING",
+ "LIBCLANG_OBJTRACKING",
+ "LIBCLANG_LOGGING",
+ "LIBCLANG_BGPRIO_INDEX",
+ "LIBCLANG_BGPRIO_EDIT",
+ "LIBCLANG_NOTHREADS",
+ "LIBCLANG_RESOURCE_USAGE",
+ "LIBCLANG_CODE_COMPLETION_LOGGING",
+ "INCLUDE",
+]
+
+for name in possibly_dangerous_env_vars:
+ if name in llvm_config.config.environment:
+ del llvm_config.config.environment[name]
+
+# Propagate some variables from the host environment.
+llvm_config.with_system_environment(
+ [
+ "PATH",
+ ]
+)
+
+# Take into account extra system environment variables if provided via parameter.
+if config.extra_system_environment:
+ lit_config.note(
+ "Extra system variables to propagate value from: "
+ + config.extra_system_environment
+ )
+ extra_env_vars = config.extra_system_environment.split(",")
+ for var in extra_env_vars:
+ if var in os.environ:
+ llvm_config.with_system_environment(var)
+
+llvm_config.with_environment("PATH", config.lit_tools_dir, append_path=True)
+
+# Configure LD_LIBRARY_PATH
+config.available_features.add("linux")
+llvm_config.with_system_environment(
+ ["LD_LIBRARY_PATH", "LIBRARY_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH"]
+)
+llvm_config.with_environment(
+ "LD_LIBRARY_PATH", config.libsycl_libs_dir, append_path=True
+)
+
+llvm_config.with_environment("PATH", config.libsycl_tools_dir, append_path=True)
+
+if config.extra_environment:
+ lit_config.note("Extra environment variables")
+ for env_pair in config.extra_environment.split(","):
+ [var, val] = env_pair.split("=", 1)
+ if val:
+ llvm_config.with_environment(var, val)
+ lit_config.note("\t" + var + "=" + val)
+ else:
+ lit_config.note("\tUnset " + var)
+ llvm_config.with_environment(var, "")
+
+
+# Temporarily modify environment to be the same that we use when running tests
+class test_env:
+ def __enter__(self):
+ self.old_environ = dict(os.environ)
+ os.environ.clear()
+ os.environ.update(config.environment)
+ self.old_dir = os.getcwd()
+ os.chdir(config.libsycl_obj_root)
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ os.environ.clear()
+ os.environ.update(self.old_environ)
+ os.chdir(self.old_dir)
+
+
+# General substritutions
+config.substitutions.append(
+ (
+ "%sycl_options",
+ " -lsycl"
+ + " -isystem "
+ + config.libsycl_include
+ + " -isystem "
+ + os.path.join(config.libsycl_include, "sycl")
+ + " -L"
+ + config.libsycl_libs_dir,
+ )
+)
+config.substitutions.append(("%sycl_libs_dir", config.libsycl_libs_dir))
+config.substitutions.append(("%sycl_static_libs_dir", config.libsycl_libs_dir))
+config.substitutions.append(("%obj_ext", ".o"))
+config.substitutions.append(("%sycl_include", "-isystem " + config.libsycl_include))
+config.substitutions.append(("%include_option", "-include"))
+config.substitutions.append(("%debug_option", "-g"))
+config.substitutions.append(("%cxx_std_option", "-std="))
+config.substitutions.append(("%fPIC", "-fPIC"))
+config.substitutions.append(("%shared_lib", "-shared"))
+config.substitutions.append(("%O0", "-O0"))
+
+sycl_ls = FindTool("sycl-ls").resolve(
+ llvm_config, os.pathsep.join([config.libsycl_bin_dir, config.llvm_tools_dir])
+)
+if not sycl_ls:
+ lit_config.fatal("can't find `sycl-ls`")
+
+tools = [
+ ToolSubst("FileCheck", unresolved="ignore"),
+ # not is only substituted in certain circumstances; this is lit's default
+ # behaviour.
+ ToolSubst(
+ r"\| \bnot\b", command=FindTool("not"), verbatim=True, unresolved="ignore"
+ ),
+ ToolSubst("sycl-ls", command=sycl_ls, unresolved="fatal"),
+]
+
+# Try and find each of these tools in the libsycl bin directory, in the llvm tools directory
+# or the PATH, in that order. If found, they will be added as substitutions with the full path
+# to the tool.
+llvm_config.add_tool_substitutions(
+ tools, [config.libsycl_bin_dir, config.llvm_tools_dir, os.environ.get("PATH", "")]
+)
+
+lit_config.note("Targeted devices: {}".format(", ".join(config.libsycl_devices)))
+with test_env():
+ sycl_ls_output = subprocess.check_output(sycl_ls, text=True, shell=True)
+
+ if len(config.libsycl_devices) == 1 and config.libsycl_devices[0] == "all":
+ devices = set()
+ for line in sycl_ls_output.splitlines():
+ if not line.startswith("["):
+ continue
+ backend, device = line[1:].split("]")[0].split(":")
+ devices.add("{}:{}".format(backend, device))
+ config.libsycl_devices = list(devices)
+
+if len(config.libsycl_devices) == 0:
+ lit_config.error("No sycl devices available.")
+
+if len(config.libsycl_devices) > 1:
+ lit_config.note(
+ "Running on multiple devices, XFAIL-marked tests will be skipped on corresponding devices."
+ )
+
+available_devices = {
+ "level_zero": "gpu",
+}
+for d in config.libsycl_devices:
+ be, dev = d.split(":")
+ if be not in available_devices:
+ lit_config.error("Unsupported device {}".format(d))
+ if dev not in available_devices[be]:
+ lit_config.error("Unsupported device {}".format(d))
+
+for sycl_device in config.libsycl_devices:
+ be, dev = sycl_device.split(":")
+ config.available_features.add("any-device-is-" + dev)
+ config.available_features.add("any-device-is-" + be)
+
+# Check if user passed verbose-print parameter, if yes, add VERBOSE_PRINT macro
+if "verbose-print" in lit_config.params:
+ verbose_print = "-DVERBOSE_PRINT"
+else:
+ verbose_print = ""
+
+clangxx = " " + config.libsycl_compiler + " -Werror " + config.cxx_flags + verbose_print
+config.substitutions.append(("%clangxx", clangxx))
+
+config.test_format = lit.formats.ShTest()
+
+try:
+ import psutil
+
+ # Set timeout for a single test
+ lit_config.maxIndividualTestTime = 600
+
+except ImportError:
+ pass
diff --git a/libsycl/test_e2e/lit.site.cfg.py.in b/libsycl/test_e2e/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..0e21628272dfa
--- /dev/null
+++ b/libsycl/test_e2e/lit.site.cfg.py.in
@@ -0,0 +1,39 @@
+ at LIT_SITE_CFG_IN_HEADER@
+
+import subprocess
+import site
+
+site.addsitedir("@CMAKE_CURRENT_SOURCE_DIR@")
+
+config.libsycl_compiler = lit_config.params.get("libsycl_compiler", "@LIBSYCL_CXX_COMPILER@")
+config.libsycl_root_dir= os.path.dirname(os.path.dirname(config.libsycl_compiler))
+config.libsycl_bin_dir = os.path.join(config.libsycl_root_dir, 'bin')
+
+config.cxx_flags = lit_config.params.get("cxx_flags", "@LIBSYCL_E2E_CLANG_CXX_FLAGS@")
+
+config.extra_environment = lit_config.params.get("extra_environment", "@LIT_EXTRA_ENVIRONMENT@")
+config.extra_system_environment = lit_config.params.get("extra_system_environment", "@LIT_EXTRA_SYSTEM_ENVIRONMENT@")
+
+def get_libsycl_tool_path(name):
+ try:
+ return subprocess.check_output([config.libsycl_compiler, "-print-prog-name=" + name], text=True)
+ except subprocess.CalledProcessError:
+ return os.path.join(config.libsycl_bin_dir, name)
+
+config.llvm_tools_dir = os.path.dirname(get_libsycl_tool_path("llvm-config"))
+config.lit_tools_dir = os.path.dirname("@TEST_SUITE_LIT@")
+
+config.libsycl_tools_dir = config.llvm_tools_dir
+config.libsycl_include = os.path.join(config.libsycl_root_dir, 'include')
+config.libsycl_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
+config.libsycl_libs_dir = os.path.join(config.libsycl_root_dir, 'lib/x86_64-unknown-linux-gnu')
+
+config.level_zero_libs_dir = "@LEVEL_ZERO_LIBS_DIR@"
+config.level_zero_include = "@LEVEL_ZERO_INCLUDE@"
+
+config.libsycl_devices = lit_config.params.get("libsycl_devices", "@LIBSYCL_TEST_E2E_TARGETS@").split(';')
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")
More information about the llvm-commits
mailing list