RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-31 20:51:52 +01:00
Code Issues Projects Releases Wiki Activity

[lit] Improve consistency when printing test results

Browse Source
This commit is contained in:
Julian Lettner 2019-02-27 21:46:04 -08:00 committed by Julian Lettner
parent 3fce01ebae
commit da1f80b302
2 changed files with 51 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
utils/lit/lit/main.py
View File

@ -238,55 +238,59 @@ def execute_in_tmp_dir(run, lit_config):
# FIXME: Re-try after timeout on Windows.
lit_config.warning("Failed to delete temp directory '%s'" % tmp_dir)
def print_summary(tests, elapsed, opts):
if not opts.quiet:
print('\nTesting Time: %.2fs' % elapsed)
# Status code, summary label, group label
groups = [
# Successes
(lit.Test.UNSUPPORTED, 'Unsupported Tests ', 'Unsupported'),
(lit.Test.PASS, 'Expected Passes ', ''),
(lit.Test.FLAKYPASS, 'Passes With Retry ', ''),
(lit.Test.XFAIL, 'Expected Failures ', 'Expected Failing'),
# Failures
(lit.Test.UNRESOLVED, 'Unresolved Tests ', 'Unresolved'),
(lit.Test.TIMEOUT, 'Individual Timeouts', 'Timed Out'),
(lit.Test.FAIL, 'Unexpected Failures', 'Failing'),
(lit.Test.XPASS, 'Unexpected Passes ', 'Unexpected Passing')]
byCode = {}
by_code = {code: [] for (code, _, _) in groups}
for test in tests:
if test.result.code not in byCode:
byCode[test.result.code] = []
byCode[test.result.code].append(test)
by_code[test.result.code].append(test)
# Print each test in any of the failing groups.
for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
('Failing Tests', lit.Test.FAIL),
('Unresolved Tests', lit.Test.UNRESOLVED),
('Unsupported Tests', lit.Test.UNSUPPORTED),
('Expected Failing Tests', lit.Test.XFAIL),
('Timed Out Tests', lit.Test.TIMEOUT)):
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
(lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
(lit.Test.UNRESOLVED == code and (opts.max_failures is not None)):
continue
elts = byCode.get(code)
if not elts:
continue
print('*'*20)
print('%s (%d):' % (title, len(elts)))
for test in elts:
print(' %s' % test.getFullName())
sys.stdout.write('\n')
for (code, _, group_label) in groups:
print_group(code, group_label, by_code[code], opts)
if opts.timeTests and tests:
# Order by time.
test_times = [(test.getFullName(), test.result.elapsed)
for test in tests]
test_times = [(t.getFullName(), t.result.elapsed) for t in tests]
lit.util.printHistogram(test_times, title='Tests')
for name,code in (('Expected Passes ', lit.Test.PASS),
('Passes With Retry ', lit.Test.FLAKYPASS),
('Expected Failures ', lit.Test.XFAIL),
('Unsupported Tests ', lit.Test.UNSUPPORTED),
('Unresolved Tests ', lit.Test.UNRESOLVED),
('Unexpected Passes ', lit.Test.XPASS),
('Unexpected Failures', lit.Test.FAIL),
('Individual Timeouts', lit.Test.TIMEOUT)):
if opts.quiet and not code.isFailure:
continue
N = len(byCode.get(code,[]))
if N:
print(' %s: %d' % (name,N))
if not opts.quiet:
print('\nTesting Time: %.2fs' % elapsed)
for (code, summary_label, _) in groups:
print_group_summary(code, summary_label, by_code[code], opts.quiet)
def print_group(code, label, tests, opts):
if not tests:
return
if code == lit.Test.PASS:
return
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
(lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
(lit.Test.UNRESOLVED == code and (opts.max_failures is not None)):
return
print('*' * 20)
print('%s Tests (%d):' % (label, len(tests)))
for test in tests:
print(' %s' % test.getFullName())
sys.stdout.write('\n')
def print_group_summary(code, label, tests, quiet):
count = len(tests)
if count and (code.isFailure or not quiet):
print(' %s: %d' % (label, count))
def write_test_results(tests, lit_config, elapsed, output_path):
# TODO(yln): audit: unexecuted tests

12
utils/lit/tests/shtest-format.py
View File

@ -68,22 +68,22 @@
# CHECK-NEXT: --
# CHECK-NEXT: true
# CHECK-NEXT: --
# CHECK: Testing Time
# CHECK: Unexpected Passing Tests (1)
# CHECK: shtest-format :: xpass.txt
# CHECK: Failing Tests (3)
# CHECK: shtest-format :: external_shell/fail.txt
# CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
# CHECK: shtest-format :: fail.txt
# CHECK: Unexpected Passing Tests (1)
# CHECK: shtest-format :: xpass.txt
# CHECK: Testing Time:
# CHECK: Unsupported Tests : 4
# CHECK: Expected Passes : 7
# CHECK: Expected Failures : 4
# CHECK: Unsupported Tests : 4
# CHECK: Unresolved Tests : 3
# CHECK: Unexpected Passes : 1
# CHECK: Unexpected Failures: 3
# CHECK: Unexpected Passes : 1
# XUNIT: <?xml version="1.0" encoding="UTF-8" ?>