[llvm] r309310 - Un-revert "Teach the CMake build system to run lit's test suite. These can be run"

[Yung, Douglas via llvm-commits]

Hi Brian,

This change seems to be tripping up our internal Windows build bot. When running the lit test suite after building, several of the tests fail because they cannot find FileCheck. What is weird is that our upstream public bot was able to successfully run these same tests with no problem.

After some experimentation, it seems the root of the problem is that the upstream Windows build bots use Ninja (-G Ninja) to build, while our internal build bot uses Visual Studio (-G "Visual Studio 14 Win64") to build. If I run the same cmake command but changing only the generator (and building with Ninja/msbuild respectively), I can reproduce the failure that we are seeing. The error we are getting when using the Visual Studio 14 Win64 Generator is:

$ "C:\tools\python2-x86_32\python.exe" "C:/j/w/opensource/opensource_to_sce_merge/o/llvm\utils\lit\lit.py" "--help"
$ "FileCheck" "C:\j\w\opensource\opensource_to_sce_merge\build\utils\lit\tests\usage.py"
# command stderr:
'FileCheck': command not found
error: command failed with exit status: 127

For some reason, it seems that when running the tests using Ninja, the test harness is able to located the built binaries, while when not using Ninja, it is not able to. I suspect the lit.cfg file in the lit test directory needs a little more code to get this working correctly. Can you take a look please?

Douglas Yung

> -----Original Message-----
> From: llvm-commits [mailto:llvm-commits-bounces at lists.llvm.org] On Behalf Of
> Brian Gesiak via llvm-commits
> Sent: Thursday, July 27, 2017 12:19
> To: llvm-commits at lists.llvm.org
> Subject: [llvm] r309310 - Un-revert "Teach the CMake build system to run lit's
> test suite. These can be run"
> 
> Author: modocache
> Date: Thu Jul 27 12:18:35 2017
> New Revision: 309310
> 
> URL: http://llvm.org/viewvc/llvm-project?rev=309310&view=rev
> Log:
> Un-revert "Teach the CMake build system to run lit's test suite. These can be
> run"
> 
> Summary:
> Depends on https://reviews.llvm.org/D35879.
> 
> This reverts rL257268, which in turn was a revert of rL257221.
> https://reviews.llvm.org/D35879 marks the tests in the lit test suite that
> fail on Windows as XFAIL, which should allow these tests to pass on Windows-
> based buildbots.
> 
> Reviewers: delcypher, beanz, mgorny, jroelofs, rnk
> 
> Reviewed By: mgorny
> 
> Subscribers: rnk, ddunbar, george.karpenkov, llvm-commits
> 
> Differential Revision: https://reviews.llvm.org/D35880
> 
> Added:
>     llvm/trunk/utils/lit/CMakeLists.txt
>     llvm/trunk/utils/lit/tests/lit.site.cfg.in
> Modified:
>     llvm/trunk/CMakeLists.txt
>     llvm/trunk/utils/lit/tests/lit.cfg
> 
> Modified: llvm/trunk/CMakeLists.txt
> URL: http://llvm.org/viewvc/llvm-
> project/llvm/trunk/CMakeLists.txt?rev=309310&r1=309309&r2=309310&view=diff
> ==============================================================================
> --- llvm/trunk/CMakeLists.txt (original)
> +++ llvm/trunk/CMakeLists.txt Thu Jul 27 12:18:35 2017
> @@ -897,6 +897,7 @@ if( LLVM_INCLUDE_TESTS )
>        NO_INSTALL
>        ALWAYS_CLEAN)
>    endif()
> +  add_subdirectory(utils/lit)
>    add_subdirectory(test)
>    add_subdirectory(unittests)
>    if( LLVM_INCLUDE_UTILS )
> 
> Added: llvm/trunk/utils/lit/CMakeLists.txt
> URL: http://llvm.org/viewvc/llvm-
> project/llvm/trunk/utils/lit/CMakeLists.txt?rev=309310&view=auto
> ==============================================================================
> --- llvm/trunk/utils/lit/CMakeLists.txt (added)
> +++ llvm/trunk/utils/lit/CMakeLists.txt Thu Jul 27 12:18:35 2017
> @@ -0,0 +1,26 @@
> +# The configured file is not placed in the correct location # until the
> +tests are run as we need to copy it into # a copy of the tests folder
> +configure_file("tests/lit.site.cfg.in" "lit.site.cfg" @ONLY)
> +
> +# Lit's test suite creates output files next to the sources which makes
> +the # source tree dirty. This is undesirable because we do out of source
> builds.
> +# To work around this the tests and the configuration file are copied
> +into the # build directory just before running them. The tests are not
> +copied over at # configure time (i.e. `file(COPY ...)`) because this
> +could lead to stale # tests being run.
> +add_custom_target(prepare-check-lit
> +  COMMAND ${CMAKE_COMMAND} -E copy_directory
> "${CMAKE_CURRENT_SOURCE_DIR}/tests" "${CMAKE_CURRENT_BINARY_DIR}/tests"
> +  COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg"
> "${CMAKE_CURRENT_BINARY_DIR}/tests"
> +  COMMENT "Preparing lit tests"
> +)
> +
> +# Add rules for lit's own test suite
> +add_lit_testsuite(check-lit "Running lit's tests"
> +  ${CMAKE_CURRENT_BINARY_DIR}
> +  DEPENDS "FileCheck" "not" "prepare-check-lit"
> +)
> +
> +# For IDEs
> +set_target_properties(check-lit PROPERTIES FOLDER "Tests")
> +set_target_properties(prepare-check-lit PROPERTIES FOLDER "Tests")
> 
> Modified: llvm/trunk/utils/lit/tests/lit.cfg
> URL: http://llvm.org/viewvc/llvm-
> project/llvm/trunk/utils/lit/tests/lit.cfg?rev=309310&r1=309309&r2=309310&view
> =diff
> ==============================================================================
> --- llvm/trunk/utils/lit/tests/lit.cfg (original)
> +++ llvm/trunk/utils/lit/tests/lit.cfg Thu Jul 27 12:18:35 2017
> @@ -26,12 +26,20 @@ config.test_exec_root = config.test_sour
> config.target_triple = '(unused)'
> 
>  src_root = os.path.join(config.test_source_root, '..') -
> config.environment['PYTHONPATH'] = src_root
> +llvm_src_root = getattr(config, 'llvm_src_root', None) if llvm_src_root
> +!= None:
> +  # ``src_root`` may be in LLVM's binary build directory which does not
> +contain
> +  # ``lit.py``, so use `llvm_src_root` instead.
> +  lit_path = os.path.join(llvm_src_root, 'utils', 'lit')
> +else:
> +  lit_path = src_root
> +
> +config.environment['PYTHONPATH'] = lit_path # Required because some
> +tests import the lit module
>  config.substitutions.append(('%{src_root}', src_root))
> config.substitutions.append(('%{inputs}', os.path.join(
>              src_root, 'tests', 'Inputs')))
> config.substitutions.append(('%{lit}', "%%{python} %s" % (
> -            os.path.join(src_root, 'lit.py'),)))
> +            os.path.join(lit_path, 'lit.py'),)))
>  config.substitutions.append(('%{python}', sys.executable))
> 
>  # Enable coverage.py reporting, assuming the coverage module has been
> installed @@ -55,3 +63,9 @@ except ImportError:
> 
>  if sys.platform.startswith('win') or sys.platform.startswith('cygwin'):
>      config.available_features.add('windows')
> +
> +# Add llvm tools directory if this config is being loaded indirectly
> +llvm_tools_dir = getattr(config, 'llvm_tools_dir', None) if
> +llvm_tools_dir != None:
> +    path = os.path.pathsep.join((llvm_tools_dir, config.environment['PATH']))
> +    config.environment['PATH'] = path
> 
> Added: llvm/trunk/utils/lit/tests/lit.site.cfg.in
> URL: http://llvm.org/viewvc/llvm-
> project/llvm/trunk/utils/lit/tests/lit.site.cfg.in?rev=309310&view=auto
> ==============================================================================
> --- llvm/trunk/utils/lit/tests/lit.site.cfg.in (added)
> +++ llvm/trunk/utils/lit/tests/lit.site.cfg.in Thu Jul 27 12:18:35 2017
> @@ -0,0 +1,8 @@
> +## Autogenerated by LLVM/Clang configuration.
> +# Do not edit!
> +config.llvm_src_root = "@LLVM_SOURCE_DIR@"
> +config.llvm_obj_root = "@LLVM_BINARY_DIR@"
> +config.llvm_tools_dir = "@LLVM_TOOLS_BINARY_DIR@"
> +
> +# Let the main config do the real work.
> +lit_config.load_config(config,
> +"@LLVM_BINARY_DIR@/utils/lit/tests/lit.cfg")
> 
> 
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits