mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-10-19 19:12:56 +02:00
[lit] Add a --max-failures option.
- This is primarily useful as a "fail fast" mode for lit, where it will stop running tests after the first failure. - Patch by Max Moiseev. llvm-svn: 282452
This commit is contained in:
parent
836f9d2e4e
commit
52f97afa1b
@ -23,7 +23,8 @@ class LitConfig(object):
|
|||||||
useValgrind, valgrindLeakCheck, valgrindArgs,
|
useValgrind, valgrindLeakCheck, valgrindArgs,
|
||||||
noExecute, debug, isWindows,
|
noExecute, debug, isWindows,
|
||||||
params, config_prefix = None,
|
params, config_prefix = None,
|
||||||
maxIndividualTestTime = 0):
|
maxIndividualTestTime = 0,
|
||||||
|
maxFailures = None):
|
||||||
# The name of the test runner.
|
# The name of the test runner.
|
||||||
self.progname = progname
|
self.progname = progname
|
||||||
# The items to add to the PATH environment variable.
|
# The items to add to the PATH environment variable.
|
||||||
@ -60,6 +61,7 @@ class LitConfig(object):
|
|||||||
self.valgrindArgs.extend(self.valgrindUserArgs)
|
self.valgrindArgs.extend(self.valgrindUserArgs)
|
||||||
|
|
||||||
self.maxIndividualTestTime = maxIndividualTestTime
|
self.maxIndividualTestTime = maxIndividualTestTime
|
||||||
|
self.maxFailures = maxFailures
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def maxIndividualTestTime(self):
|
def maxIndividualTestTime(self):
|
||||||
|
@ -232,6 +232,9 @@ def main_with_tmp(builtinParameters):
|
|||||||
help="Maximum time to spend running a single test (in seconds)."
|
help="Maximum time to spend running a single test (in seconds)."
|
||||||
"0 means no time limit. [Default: 0]",
|
"0 means no time limit. [Default: 0]",
|
||||||
type=int, default=None)
|
type=int, default=None)
|
||||||
|
group.add_option("", "--max-failures", dest="maxFailures",
|
||||||
|
help="Stop execution after the given number of failures.",
|
||||||
|
action="store", type=int, default=None)
|
||||||
parser.add_option_group(group)
|
parser.add_option_group(group)
|
||||||
|
|
||||||
group = OptionGroup(parser, "Test Selection")
|
group = OptionGroup(parser, "Test Selection")
|
||||||
@ -291,6 +294,9 @@ def main_with_tmp(builtinParameters):
|
|||||||
else:
|
else:
|
||||||
opts.numThreads = 1
|
opts.numThreads = 1
|
||||||
|
|
||||||
|
if opts.maxFailures == 0:
|
||||||
|
parser.error("Setting --max-failures to 0 does not have any effect.")
|
||||||
|
|
||||||
inputs = args
|
inputs = args
|
||||||
|
|
||||||
# Create the user defined parameters.
|
# Create the user defined parameters.
|
||||||
@ -324,7 +330,8 @@ def main_with_tmp(builtinParameters):
|
|||||||
isWindows = isWindows,
|
isWindows = isWindows,
|
||||||
params = userParams,
|
params = userParams,
|
||||||
config_prefix = opts.configPrefix,
|
config_prefix = opts.configPrefix,
|
||||||
maxIndividualTestTime = maxIndividualTestTime)
|
maxIndividualTestTime = maxIndividualTestTime,
|
||||||
|
maxFailures = opts.maxFailures)
|
||||||
|
|
||||||
# Perform test discovery.
|
# Perform test discovery.
|
||||||
run = lit.run.Run(litConfig,
|
run = lit.run.Run(litConfig,
|
||||||
@ -477,7 +484,8 @@ def main_with_tmp(builtinParameters):
|
|||||||
('Expected Failing Tests', lit.Test.XFAIL),
|
('Expected Failing Tests', lit.Test.XFAIL),
|
||||||
('Timed Out Tests', lit.Test.TIMEOUT)):
|
('Timed Out Tests', lit.Test.TIMEOUT)):
|
||||||
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
|
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
|
||||||
(lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
|
(lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
|
||||||
|
(lit.Test.UNRESOLVED == code and (opts.maxFailures is not None)):
|
||||||
continue
|
continue
|
||||||
elts = byCode.get(code)
|
elts = byCode.get(code)
|
||||||
if not elts:
|
if not elts:
|
||||||
|
@ -152,6 +152,19 @@ def run_one_tester(run, provider, display):
|
|||||||
|
|
||||||
###
|
###
|
||||||
|
|
||||||
|
def handleFailures(provider, consumer, maxFailures):
|
||||||
|
class _Display(object):
|
||||||
|
def __init__(self, display):
|
||||||
|
self.display = display
|
||||||
|
self.maxFailures = maxFailures or object()
|
||||||
|
self.failedCount = 0
|
||||||
|
def update(self, test):
|
||||||
|
self.display.update(test)
|
||||||
|
self.failedCount += (test.result.code == lit.Test.FAIL)
|
||||||
|
if self.failedCount == self.maxFailures:
|
||||||
|
provider.cancel()
|
||||||
|
consumer.display = _Display(consumer.display)
|
||||||
|
|
||||||
class Run(object):
|
class Run(object):
|
||||||
"""
|
"""
|
||||||
This class represents a concrete, configured testing run.
|
This class represents a concrete, configured testing run.
|
||||||
@ -232,6 +245,7 @@ class Run(object):
|
|||||||
|
|
||||||
# Create the test provider.
|
# Create the test provider.
|
||||||
provider = TestProvider(queue_impl, canceled_flag)
|
provider = TestProvider(queue_impl, canceled_flag)
|
||||||
|
handleFailures(provider, consumer, self.lit_config.maxFailures)
|
||||||
|
|
||||||
# Queue the tests outside the main thread because we can't guarantee
|
# Queue the tests outside the main thread because we can't guarantee
|
||||||
# that we can put() all the tests without blocking:
|
# that we can put() all the tests without blocking:
|
||||||
|
14
utils/lit/tests/max-failures.py
Normal file
14
utils/lit/tests/max-failures.py
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
# Check the behavior of --max-failures option.
|
||||||
|
#
|
||||||
|
# RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out
|
||||||
|
# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/shtest-shell >> %t.out
|
||||||
|
# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/shtest-shell >> %t.out
|
||||||
|
# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/shtest-shell 2>> %t.out
|
||||||
|
# RUN: FileCheck < %t.out %s
|
||||||
|
#
|
||||||
|
# END.
|
||||||
|
|
||||||
|
# CHECK: Failing Tests (3)
|
||||||
|
# CHECK: Failing Tests (1)
|
||||||
|
# CHECK: Failing Tests (2)
|
||||||
|
# CHECK: error: Setting --max-failures to 0 does not have any effect.
|
Loading…
Reference in New Issue
Block a user