From 3fdd4ff2ee351eb83bbaa061f4f6754035a25f06 Mon Sep 17 00:00:00 2001 From: "Joel E. Denny" Date: Fri, 16 Jul 2021 19:13:34 -0400 Subject: [PATCH] [lit] Add --xfail-not/LIT_XFAIL_NOT For example, I need this lately in my CI config: LIT_XFAIL_NOT='libomptarget :: nvptx64-nvidia-cuda :: unified_shared_memory/api.c' That test specifies an XFAIL directive, but I get an XPASS result. Reviewed By: jhenderson Differential Revision: https://reviews.llvm.org/D106022 --- docs/CommandGuide/lit.rst | 9 ++++++++ utils/lit/lit/Test.py | 6 +++++ utils/lit/lit/cl_arguments.py | 5 ++++ utils/lit/lit/main.py | 2 ++ .../tests/Inputs/xfail-cl/a/test-xfail.txt | 2 ++ .../tests/Inputs/xfail-cl/b/test-xfail.txt | 2 ++ .../lit/tests/Inputs/xfail-cl/true-xfail.txt | 2 ++ utils/lit/tests/xfail-cl.py | 23 +++++++++++++++++-- 8 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt create mode 100644 utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt create mode 100644 utils/lit/tests/Inputs/xfail-cl/true-xfail.txt diff --git a/docs/CommandGuide/lit.rst b/docs/CommandGuide/lit.rst index dc2b57cda8a..9c48d3abb8d 100644 --- a/docs/CommandGuide/lit.rst +++ b/docs/CommandGuide/lit.rst @@ -263,6 +263,15 @@ The timing data is stored in the `test_exec_root` in a file named LIT_XFAIL="affinity/kmp-hw-subset.c;libomptarget :: x86_64-pc-linux-gnu :: offloading/memory_manager.cpp" +.. option:: --xfail-not=LIST + + Do not treat the specified tests as ``XFAIL``. The environment variable + ``LIT_XFAIL_NOT`` can also be used in place of this option. The syntax is the + same as for :option:`--xfail` and ``LIT_XFAIL``. :option:`--xfail-not` and + ``LIT_XFAIL_NOT`` always override all other ``XFAIL`` specifications, + including an :option:`--xfail` appearing later on the command line. The + primary purpose is to suppress an ``XPASS`` result without modifying a test + case that uses the ``XFAIL`` directive. ADDITIONAL OPTIONS ------------------ diff --git a/utils/lit/lit/Test.py b/utils/lit/lit/Test.py index 45ab12a85bd..77b9c235e40 100644 --- a/utils/lit/lit/Test.py +++ b/utils/lit/lit/Test.py @@ -231,6 +231,9 @@ class Test: # handlers, and will be honored when the test result is supplied. self.xfails = [] + # If true, ignore all items in self.xfails. + self.xfail_not = False + # A list of conditions that must be satisfied before running the test. # Each condition is a boolean expression of features. All of them # must be True for the test to run. @@ -309,6 +312,9 @@ class Test: Throws ValueError if an XFAIL line has a syntax error. """ + if self.xfail_not: + return False + features = self.config.available_features triple = getattr(self.suite.config, 'target_triple', "") diff --git a/utils/lit/lit/cl_arguments.py b/utils/lit/lit/cl_arguments.py index 3eb1870bf16..70e0c8d6a17 100644 --- a/utils/lit/lit/cl_arguments.py +++ b/utils/lit/lit/cl_arguments.py @@ -171,6 +171,11 @@ def parse_args(): type=_semicolon_list, help="XFAIL tests with paths in the semicolon separated list", default=os.environ.get("LIT_XFAIL", "")) + selection_group.add_argument("--xfail-not", + metavar="LIST", + type=_semicolon_list, + help="do not XFAIL tests with paths in the semicolon separated list", + default=os.environ.get("LIT_XFAIL_NOT", "")) selection_group.add_argument("--num-shards", dest="numShards", metavar="M", diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py index b60c30c6845..6924aee3d51 100755 --- a/utils/lit/lit/main.py +++ b/utils/lit/lit/main.py @@ -197,6 +197,8 @@ def mark_xfail(selected_tests, opts): test_full_name = t.getFullName() if test_file in opts.xfail or test_full_name in opts.xfail: t.xfails += '*' + if test_file in opts.xfail_not or test_full_name in opts.xfail_not: + t.xfail_not = True def mark_excluded(discovered_tests, selected_tests): excluded_tests = set(discovered_tests) - set(selected_tests) diff --git a/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt b/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt new file mode 100644 index 00000000000..b0d85523336 --- /dev/null +++ b/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt @@ -0,0 +1,2 @@ +# XFAIL: * +# RUN: true diff --git a/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt b/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt new file mode 100644 index 00000000000..f0ae3fecf2d --- /dev/null +++ b/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt @@ -0,0 +1,2 @@ +# XFAIL: * +# RUN: false diff --git a/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt b/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt new file mode 100644 index 00000000000..b0d85523336 --- /dev/null +++ b/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt @@ -0,0 +1,2 @@ +# XFAIL: * +# RUN: true diff --git a/utils/lit/tests/xfail-cl.py b/utils/lit/tests/xfail-cl.py index 1d5fdb079c2..ef1bb0414cf 100644 --- a/utils/lit/tests/xfail-cl.py +++ b/utils/lit/tests/xfail-cl.py @@ -1,16 +1,29 @@ # Check that XFAILing works via command line or env var. # RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \ -# RUN: %{inputs}/xfail-cl \ +# RUN: --xfail-not 'true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \ +# RUN: %{inputs}/xfail-cl \ # RUN: | FileCheck --check-prefix=CHECK-FILTER %s # RUN: env LIT_XFAIL='false.txt;false2.txt;top-level-suite :: b :: test.txt' \ +# RUN: LIT_XFAIL_NOT='true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \ # RUN: %{lit} %{inputs}/xfail-cl \ # RUN: | FileCheck --check-prefix=CHECK-FILTER %s +# Check that --xfail-not and LIT_XFAIL_NOT always have precedence. + +# RUN: env LIT_XFAIL=true-xfail.txt \ +# RUN: %{lit} --xfail true-xfail.txt --xfail-not true-xfail.txt \ +# RUN: --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \ +# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s + +# RUN: env LIT_XFAIL_NOT=true-xfail.txt LIT_XFAIL=true-xfail.txt \ +# RUN: %{lit} --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \ +# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s + # END. -# CHECK-FILTER: Testing: 7 tests, {{[1-7]}} workers +# CHECK-FILTER: Testing: 10 tests, {{[0-9]*}} workers # CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test.txt # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test.txt # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: a :: false.txt @@ -18,3 +31,9 @@ # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false.txt # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false2.txt # CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true.txt +# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true-xfail.txt +# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test-xfail.txt +# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test-xfail.txt + +# CHECK-OVERRIDE: Testing: 1 tests, {{[0-9]*}} workers +# CHECK-OVERRIDE: {{^}}PASS: top-level-suite :: true-xfail.txt