[lit] Add --xfail-not/LIT_XFAIL_NOT

For example, I need this lately in my CI config:

LIT_XFAIL_NOT='libomptarget :: nvptx64-nvidia-cuda :: unified_shared_memory/api.c'

That test specifies an XFAIL directive, but I get an XPASS result.

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D106022

docs/CommandGuide/lit.rst

@@ -263,6 +263,15 @@ The timing data is stored in the `test_exec_root` in a file named
 
     LIT_XFAIL="affinity/kmp-hw-subset.c;libomptarget :: x86_64-pc-linux-gnu :: offloading/memory_manager.cpp"
 
+.. option:: --xfail-not=LIST
+
+  Do not treat the specified tests as ``XFAIL``.  The environment variable
+  ``LIT_XFAIL_NOT`` can also be used in place of this option.  The syntax is the
+  same as for :option:`--xfail` and ``LIT_XFAIL``.  :option:`--xfail-not` and
+  ``LIT_XFAIL_NOT`` always override all other ``XFAIL`` specifications,
+  including an :option:`--xfail` appearing later on the command line.  The
+  primary purpose is to suppress an ``XPASS`` result without modifying a test
+  case that uses the ``XFAIL`` directive.
 
 ADDITIONAL OPTIONS
 ------------------

utils/lit/lit/Test.py

@@ -231,6 +231,9 @@ class Test:
         # handlers, and will be honored when the test result is supplied.
         self.xfails = []
 
+        # If true, ignore all items in self.xfails.
+        self.xfail_not = False
+
         # A list of conditions that must be satisfied before running the test.
         # Each condition is a boolean expression of features. All of them
         # must be True for the test to run.
@@ -309,6 +312,9 @@ class Test:
         Throws ValueError if an XFAIL line has a syntax error.
         """
 
+        if self.xfail_not:
+          return False
+
         features = self.config.available_features
         triple = getattr(self.suite.config, 'target_triple', "")
 

utils/lit/lit/cl_arguments.py

@@ -171,6 +171,11 @@ def parse_args():
             type=_semicolon_list,
             help="XFAIL tests with paths in the semicolon separated list",
             default=os.environ.get("LIT_XFAIL", ""))
+    selection_group.add_argument("--xfail-not",
+            metavar="LIST",
+            type=_semicolon_list,
+            help="do not XFAIL tests with paths in the semicolon separated list",
+            default=os.environ.get("LIT_XFAIL_NOT", ""))
     selection_group.add_argument("--num-shards",
             dest="numShards",
             metavar="M",

utils/lit/lit/main.py

@@ -197,6 +197,8 @@ def mark_xfail(selected_tests, opts):
         test_full_name = t.getFullName()
         if test_file in opts.xfail or test_full_name in opts.xfail:
             t.xfails += '*'
+        if test_file in opts.xfail_not or test_full_name in opts.xfail_not:
+            t.xfail_not = True
 
 def mark_excluded(discovered_tests, selected_tests):
     excluded_tests = set(discovered_tests) - set(selected_tests)

utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt

@@ -0,0 +1,2 @@
+# XFAIL: *
+# RUN: true

utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt

@@ -0,0 +1,2 @@
+# XFAIL: *
+# RUN: false

utils/lit/tests/Inputs/xfail-cl/true-xfail.txt

@@ -0,0 +1,2 @@
+# XFAIL: *
+# RUN: true

utils/lit/tests/xfail-cl.py

@@ -1,16 +1,29 @@
 # Check that XFAILing works via command line or env var.
 
 # RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \
-# RUN:        %{inputs}/xfail-cl \
+# RUN:   --xfail-not 'true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
+# RUN:   %{inputs}/xfail-cl \
 # RUN: | FileCheck --check-prefix=CHECK-FILTER %s
 
 # RUN: env LIT_XFAIL='false.txt;false2.txt;top-level-suite :: b :: test.txt' \
+# RUN:   LIT_XFAIL_NOT='true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
 # RUN: %{lit} %{inputs}/xfail-cl \
 # RUN: | FileCheck --check-prefix=CHECK-FILTER %s
 
+# Check that --xfail-not and LIT_XFAIL_NOT always have precedence.
+
+# RUN: env LIT_XFAIL=true-xfail.txt \
+# RUN: %{lit} --xfail true-xfail.txt --xfail-not true-xfail.txt \
+# RUN:   --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \
+# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s
+
+# RUN: env LIT_XFAIL_NOT=true-xfail.txt LIT_XFAIL=true-xfail.txt \
+# RUN: %{lit} --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \
+# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s
+
 # END.
 
-# CHECK-FILTER: Testing: 7 tests, {{[1-7]}} workers
+# CHECK-FILTER: Testing: 10 tests, {{[0-9]*}} workers
 # CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: a :: false.txt
@@ -18,3 +31,9 @@
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false2.txt
 # CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true.txt
+# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true-xfail.txt
+# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test-xfail.txt
+# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test-xfail.txt
+
+# CHECK-OVERRIDE: Testing: 1 tests, {{[0-9]*}} workers
+# CHECK-OVERRIDE: {{^}}PASS: top-level-suite :: true-xfail.txt