RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-31 20:51:52 +01:00
Code Issues Projects Releases Wiki Activity

[lit] Clean output directories before running tests.

Browse Source 
Presently lit leaks files in the tests' output directories.
Specifically, if a test creates output files, lit makes no
effort to remove them prior to the next test run.  This is
problematic because it leads to false positives whenever a
test passes because stale  files were present.  In general
it is a source of flakiness that should be removed.

This patch addresses this by building the list of all test
directories that are part of the current run set, and then
deleting those directories and recreating them anew.  This
gives each test a clean baseline to start from.

Differential Revision: https://reviews.llvm.org/D34732

llvm-svn: 306832
This commit is contained in:
Zachary Turner 2017-06-30 16:01:30 +00:00
parent 06d35c8a0b
commit 8e4d247b07
15 changed files with 104 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.cpp Normal file
View File

@ -0,0 +1,7 @@
void f1();
__attribute__((always_inline)) void f2() {
f1();
}
void f3() {
f2();
}

BIN
test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.dwo
View File

Binary file not shown.

BIN
test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.o
View File

Binary file not shown.

BIN
test/DebugInfo/Inputs/split-dwarf-multiple-cu.dwo
View File

Binary file not shown.

BIN
test/DebugInfo/Inputs/split-dwarf-multiple-cu.o
View File

Binary file not shown.

2
test/DebugInfo/Inputs/split-dwarf-multiple-cu1.cpp Normal file
View File

@ -0,0 +1,2 @@
extern int i;
int i;

7
test/DebugInfo/Inputs/split-dwarf-multiple-cu2.cpp Normal file
View File

@ -0,0 +1,7 @@
void f1();
inline __attribute__((always_inline)) void f2() {
f1();
}
void f3() {
f2();
}

BIN
test/DebugInfo/Inputs/split-dwarf-test
View File

Binary file not shown.

BIN
test/DebugInfo/Inputs/split-dwarf-test.dwo
View File

Binary file not shown.

8
test/DebugInfo/llvm-symbolizer.test
View File

@ -20,8 +20,8 @@ RUN: echo "%p/Inputs/llvm-symbolizer-dwo-test 0x400514" >> %t.input
RUN: echo "%p/Inputs/fission-ranges.elf-x86_64 0x720" >> %t.input
RUN: echo "%p/Inputs/arange-overlap.elf-x86_64 0x714" >> %t.input
RUN: cp %p/Inputs/split-dwarf-test.dwo %T
RUN: echo "%p/Inputs/split-dwarf-test 0x4005d4" >> %t.input
RUN: echo "%p/Inputs/split-dwarf-test 0x4005c4" >> %t.input
RUN: echo "%p/Inputs/split-dwarf-test 0x400504" >> %t.input
RUN: echo "%p/Inputs/split-dwarf-test 0x4004f0" >> %t.input
RUN: echo "%p/Inputs/cross-cu-inlining.x86_64-macho.o 0x17" >> %t.input
RUN: cp %p/Inputs/split-dwarf-multiple-cu.dwo %T
RUN: echo "%p/Inputs/split-dwarf-multiple-cu.o 0x4" >> %t.input
@ -141,9 +141,9 @@ CHECK-NEXT: main
CHECK-NEXT: /tmp{{[/\\]}}cross-cu-inlining.c:11:0
CHECK: f2
CHECK-NEXT: b.cpp:3:3
CHECK-NEXT: split-dwarf-multiple-cu2.cpp:3:3
CHECK-NEXT: f3
CHECK-NEXT: b.cpp:6:0
CHECK-NEXT: split-dwarf-multiple-cu2.cpp:6:0
CHECK: f2
CHECK-NEXT: split-dwarf-addr-object-relocation.cpp:3:3

56
test/Other/lit-globbing.ll
View File

@ -1,28 +1,28 @@
RUN: echo TA > %T/TA.txt
RUN: echo TB > %T/TB.txt
RUN: echo TAB > %T/TAB.txt
RUN: echo %T/TA* | FileCheck -check-prefix=STAR %s
RUN: echo %T/'TA'* | FileCheck -check-prefix=STAR %s
RUN: echo %T/T'A'* | FileCheck -check-prefix=STAR %s
RUN: echo %T/T?.txt | FileCheck -check-prefix=QUESTION %s
RUN: echo %T/'T'?.txt | FileCheck -check-prefix=QUESTION %s
RUN: echo %T/T??.txt | FileCheck -check-prefix=QUESTION2 %s
RUN: echo %T/'T'??.txt | FileCheck -check-prefix=QUESTION2 %s
RUN: echo 'T*' 'T?.txt' 'T??.txt' | FileCheck -check-prefix=QUOTEDARGS %s
STAR-NOT: TB.txt
STAR: {{(TA.txt.*TAB.txt|TAB.txt.*TA.txt)}}
QUESTION-NOT: TAB.txt
QUESTION: {{(TA.txt.*TB.txt|TB.txt.*TA.txt)}}
QUESTION2-NOT: TA.txt
QUESTION2-NOT: TB.txt
QUESTION2: TAB.txt
QUOTEDARGS-NOT: .txt
QUOTEDARGS: T* T?.txt T??.txt
RUN: echo XXA > %T/XXA.txt
RUN: echo XXB > %T/XXB.txt
RUN: echo XXAB > %T/XXAB.txt
RUN: echo %T/XXA* | FileCheck -check-prefix=STAR %s
RUN: echo %T/'XXA'* | FileCheck -check-prefix=STAR %s
RUN: echo %T/XX'A'* | FileCheck -check-prefix=STAR %s
RUN: echo %T/XX?.txt | FileCheck -check-prefix=QUESTION %s
RUN: echo %T/'XX'?.txt | FileCheck -check-prefix=QUESTION %s
RUN: echo %T/XX??.txt | FileCheck -check-prefix=QUESTION2 %s
RUN: echo %T/'XX'??.txt | FileCheck -check-prefix=QUESTION2 %s
RUN: echo 'XX*' 'XX?.txt' 'XX??.txt' | FileCheck -check-prefix=QUOTEDARGS %s
STAR-NOT: XXB.txt
STAR: {{(XXA.txt.*XXAB.txt|XXAB.txt.*XXA.txt)}}
QUESTION-NOT: XXAB.txt
QUESTION: {{(XXA.txt.*XXB.txt|XXB.txt.*XXA.txt)}}
QUESTION2-NOT: XXA.txt
QUESTION2-NOT: XXB.txt
QUESTION2: XXAB.txt
QUOTEDARGS-NOT: .txt
QUOTEDARGS: XX* XX?.txt XX??.txt

8
test/Unit/lit.cfg
View File

@ -39,9 +39,11 @@ for symbolizer in ['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']:
config.environment[symbolizer] = os.environ[symbolizer]
# Win32 seeks DLLs along %PATH%.
if sys.platform in ['win32', 'cygwin'] and os.path.isdir(config.shlibdir):
config.environment['PATH'] = os.path.pathsep.join((
config.shlibdir, config.environment['PATH']))
if sys.platform in ['win32', 'cygwin']:
shlibdir = getattr(config, 'shlibdir', None)
if shlibdir is not None and os.path.isdir(shlibdir):
config.environment['PATH'] = os.path.pathsep.join((
config.shlibdir, config.environment['PATH']))
# Win32 may use %SYSTEMDRIVE% during file system shell operations, so propogate.
if sys.platform == 'win32' and 'SYSTEMDRIVE' in os.environ:

16
utils/lit/lit/Test.py
View File

@ -172,7 +172,7 @@ class TestSuite:
return os.path.join(self.source_root, *components)
def getExecPath(self, components):
return os.path.join(self.exec_root, *components)
return os.path.join(self.exec_root, "Output", *components)
class Test:
"""Test - Information on a single test instance."""
@ -222,10 +222,13 @@ class Test:
# Syntax error in an XFAIL line.
self.result.code = UNRESOLVED
self.result.output = str(e)
def getFullName(self):
return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
def getTestBaseName(self):
return self.path_in_suite[-1]
def getFilePath(self):
if self.file_path:
return self.file_path
@ -234,8 +237,11 @@ class Test:
def getSourcePath(self):
return self.suite.getSourcePath(self.path_in_suite)
def getExecPath(self):
return self.suite.getExecPath(self.path_in_suite)
def getTempFilePrefix(self):
return self.suite.getExecPath(self.path_in_suite) + ".tmp"
def getTempFileDir(self):
return os.path.dirname(self.getTempFilePrefix())
def isExpectedToFail(self):
"""
@ -347,7 +353,7 @@ class Test:
safe_name = self.suite.name.replace(".","-")
if safe_test_path:
class_name = safe_name + "." + "/".join(safe_test_path)
class_name = safe_name + "." + "/".join(safe_test_path)
else:
class_name = safe_name + "." + safe_name

54
utils/lit/lit/TestRunner.py
View File

@ -123,7 +123,7 @@ class ShellCommandResult(object):
self.exitCode = exitCode
self.timeoutReached = timeoutReached
self.outputFiles = list(outputFiles)
def executeShCmd(cmd, shenv, results, timeout=0):
"""
Wrapper around _executeShCmd that handles
@ -501,7 +501,7 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
data = None
if data is not None:
output_files.append((name, path, data))
results.append(ShellCommandResult(
cmd.commands[i], out, err, res, timeoutHelper.timeoutReached(),
output_files))
@ -573,7 +573,7 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
else:
out += data
out += "\n"
if result.stdout.strip():
out += '# command output:\n%s\n' % (result.stdout,)
if result.stderr.strip():
@ -690,37 +690,28 @@ def parseIntegratedTestScriptCommands(source_path, keywords):
finally:
f.close()
def getTempPaths(test):
"""Get the temporary location, this is always relative to the test suite
root, not test source root."""
execpath = test.getExecPath()
execdir,execbase = os.path.split(execpath)
tmpDir = os.path.join(execdir, 'Output')
tmpBase = os.path.join(tmpDir, execbase)
return tmpDir, tmpBase
def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
def getDefaultSubstitutions(test, normalize_slashes=False):
sourcepath = test.getSourcePath()
sourcedir = os.path.dirname(sourcepath)
tmpDir = test.getTempFileDir()
tmpPrefix = test.getTempFilePrefix()
baseName = test.getTestBaseName()
# Normalize slashes, if requested.
if normalize_slashes:
sourcepath = sourcepath.replace('\\', '/')
sourcedir = sourcedir.replace('\\', '/')
tmpDir = tmpDir.replace('\\', '/')
tmpBase = tmpBase.replace('\\', '/')
# We use #_MARKER_# to hide %% while we do the other substitutions.
substitutions = []
substitutions.extend([('%%', '#_MARKER_#')])
substitutions.extend(test.config.substitutions)
tmpName = tmpBase + '.tmp'
baseName = os.path.basename(tmpBase)
substitutions.extend([('%s', sourcepath),
('%S', sourcedir),
('%p', sourcedir),
('%{pathsep}', os.pathsep),
('%t', tmpName),
('%t', tmpPrefix),
('%basename_t', baseName),
('%T', tmpDir),
('#_MARKER_#', '%')])
@ -730,7 +721,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
('%/s', sourcepath.replace('\\', '/')),
('%/S', sourcedir.replace('\\', '/')),
('%/p', sourcedir.replace('\\', '/')),
('%/t', tmpBase.replace('\\', '/') + '.tmp'),
('%/t', tmpPrefix.replace('\\', '/')),
('%/T', tmpDir.replace('\\', '/')),
])
@ -740,7 +731,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
('%:s', re.sub(r'^(.):', r'\1', sourcepath)),
('%:S', re.sub(r'^(.):', r'\1', sourcedir)),
('%:p', re.sub(r'^(.):', r'\1', sourcedir)),
('%:t', re.sub(r'^(.):', r'\1', tmpBase) + '.tmp'),
('%:t', re.sub(r'^(.):', r'\1', tmpPrefix)),
('%:T', re.sub(r'^(.):', r'\1', tmpDir)),
])
else:
@ -748,7 +739,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
('%:s', sourcepath),
('%:S', sourcedir),
('%:p', sourcedir),
('%:t', tmpBase + '.tmp'),
('%:t', tmpPrefix),
('%:T', tmpDir),
])
return substitutions
@ -779,7 +770,7 @@ class ParserKind(object):
TAG: A keyword taking no value. Ex 'END.'
COMMAND: A keyword taking a list of shell commands. Ex 'RUN:'
LIST: A keyword taking a comma-separated list of values.
BOOLEAN_EXPR: A keyword taking a comma-separated list of
BOOLEAN_EXPR: A keyword taking a comma-separated list of
boolean expressions. Ex 'XFAIL:'
CUSTOM: A keyword with custom parsing semantics.
"""
@ -951,14 +942,14 @@ def parseIntegratedTestScript(test, additional_parsers=[],
IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR,
initial_value=test.requires),
IntegratedTestKeywordParser('REQUIRES-ANY:', ParserKind.CUSTOM,
IntegratedTestKeywordParser._handleRequiresAny,
initial_value=test.requires),
IntegratedTestKeywordParser._handleRequiresAny,
initial_value=test.requires),
IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
initial_value=test.unsupported),
IntegratedTestKeywordParser('END.', ParserKind.TAG)
]
keyword_parsers = {p.keyword: p for p in builtin_parsers}
# Install user-defined additional parsers.
for parser in additional_parsers:
if not isinstance(parser, IntegratedTestKeywordParser):
@ -968,7 +959,7 @@ def parseIntegratedTestScript(test, additional_parsers=[],
raise ValueError("Parser for keyword '%s' already exists"
% parser.keyword)
keyword_parsers[parser.keyword] = parser
# Collect the test lines from the script.
sourcepath = test.getSourcePath()
for line_number, command_type, ln in \
@ -1014,12 +1005,8 @@ def parseIntegratedTestScript(test, additional_parsers=[],
return script
def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
# Create the output directory if it does not already exist.
lit.util.mkdir_p(os.path.dirname(tmpBase))
execdir = os.path.dirname(test.getExecPath())
execdir = os.path.dirname(test.getTempFileDir())
if useExternalSh:
res = executeScript(test, litConfig, tmpBase, script, execdir)
else:
@ -1063,10 +1050,8 @@ def executeShTest(test, litConfig, useExternalSh,
return script
if litConfig.noExecute:
return lit.Test.Result(Test.PASS)
tmpDir, tmpBase = getTempPaths(test)
substitutions = list(extra_substitutions)
substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
substitutions += getDefaultSubstitutions(test,
normalize_slashes=useExternalSh)
script = applySubstitutions(script, substitutions)
@ -1075,7 +1060,8 @@ def executeShTest(test, litConfig, useExternalSh,
if hasattr(test.config, 'test_retry_attempts'):
attempts += test.config.test_retry_attempts
for i in range(attempts):
res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
res = _runShTest(test, litConfig, useExternalSh, script,
test.getTempFilePrefix())
if res.code != Test.FAIL:
break
# If we had to run the test more than once, count it as a flaky pass. These

20
utils/lit/lit/run.py
View File

@ -1,4 +1,5 @@
import os
import shutil
import sys
import threading
import time
@ -74,6 +75,25 @@ class Run(object):
if not self.tests or jobs == 0:
return
# Create fresh output directories for each test we're going to run.
# This guarantees that test runs will not remnants of previous test
# runs' output.
clean_paths = set()
for test in self.tests:
clean_paths.add(os.path.normpath(test.getTempFileDir()))
clean_paths = list(clean_paths)
# Sort by number of path components, to ensure that parent directories
# get deleted and re-created before child directories.
clean_paths.sort(key=lambda x: len(x.split(os.sep)))
for base in clean_paths:
if os.path.exists(base):
if not os.path.islink(base) and os.path.isdir(base):
shutil.rmtree(base, True)
else:
os.unlink(os.path)
if not os.path.exists(base):
lit.util.mkdir_p(base)
# Set up semaphores to limit parallelism of certain classes of tests.
# For example, some ASan tests require lots of virtual memory and run
# faster with less parallelism on OS X.