diff --git a/utils/UpdateTestChecks/common.py b/utils/UpdateTestChecks/common.py index 9abb8d790c5..7b4a9b3e6d1 100644 --- a/utils/UpdateTestChecks/common.py +++ b/utils/UpdateTestChecks/common.py @@ -1,9 +1,10 @@ from __future__ import print_function + +import copy +import glob import re -import string import subprocess import sys -import copy if sys.version_info[0] > 2: class string: @@ -21,11 +22,85 @@ def parse_commandline_args(parser): help='Show verbose output') parser.add_argument('-u', '--update-only', action='store_true', help='Only update test if it was already autogened') + parser.add_argument('--force-update', action='store_true', + help='Update test even if it was autogened by a different script') + parser.add_argument('--enable', action='store_true', dest='enabled', default=True, + help='Activate CHECK line generation from this point forward') + parser.add_argument('--disable', action='store_false', dest='enabled', + help='Deactivate CHECK line generation from this point forward') args = parser.parse_args() global _verbose _verbose = args.verbose return args + +class InputLineInfo(object): + def __init__(self, line, line_number, args, argv): + self.line = line + self.line_number = line_number + self.args = args + self.argv = argv + + +class TestInfo(object): + def __init__(self, test, parser, script_name, input_lines, args, argv, + comment_prefix): + self.parser = parser + self.path = test + self.args = args + self.argv = argv + self.input_lines = input_lines + self.run_lines = find_run_lines(test, self.input_lines) + self.comment_prefix = comment_prefix + if self.comment_prefix is None: + if self.path.endswith('.mir'): + self.comment_prefix = '#' + else: + self.comment_prefix = ';' + self.autogenerated_note_prefix = self.comment_prefix + ' ' + UTC_ADVERT + self.test_autogenerated_note = self.autogenerated_note_prefix + script_name + self.test_autogenerated_note += get_autogennote_suffix(parser, self.args) + + def iterlines(self, output_lines): + output_lines.append(self.test_autogenerated_note) + for line_num, input_line in enumerate(self.input_lines): + # Discard any previous script advertising. + if input_line.startswith(self.autogenerated_note_prefix): + continue + self.args, self.argv = check_for_command(input_line, self.parser, + self.args, self.argv) + if not self.args.enabled: + output_lines.append(input_line) + continue + yield InputLineInfo(input_line, line_num, self.args, self.argv) + + +def itertests(test_patterns, parser, script_name, comment_prefix=None): + for pattern in test_patterns: + # On Windows we must expand the patterns ourselves. + tests_list = glob.glob(pattern) + if not tests_list: + warn("Test file pattern '%s' was not found. Ignoring it." % (pattern,)) + continue + for test in tests_list: + with open(test) as f: + input_lines = [l.rstrip() for l in f] + args = parser.parse_args() + argv = sys.argv[:] + first_line = input_lines[0] if input_lines else "" + if UTC_ADVERT in first_line: + if script_name not in first_line and not args.force_update: + warn("Skipping test which wasn't autogenerated by " + script_name, test) + continue + args, argv = check_for_command(first_line, parser, args, argv) + elif args.update_only: + assert UTC_ADVERT not in first_line + warn("Skipping test which isn't autogenerated: " + test) + continue + yield TestInfo(test, parser, script_name, input_lines, args, argv, + comment_prefix) + + def should_add_line_to_output(input_line, prefix_set): # Skip any blank comment lines in the IR. if input_line.strip() == ';': @@ -57,7 +132,6 @@ def invoke_tool(exe, cmd_args, ir): return stdout.replace('\r\n', '\n') ##### LLVM IR parser - RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$') CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)') PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$') @@ -65,6 +139,7 @@ CHECK_RE = re.compile(r'^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAM UTC_ARGS_KEY = 'UTC_ARGS:' UTC_ARGS_CMD = re.compile(r'.*' + UTC_ARGS_KEY + '\s*(?P.*)\s*$') +UTC_ADVERT = 'NOTE: Assertions have been autogenerated by ' OPT_FUNCTION_RE = re.compile( r'^\s*define\s+(?:internal\s+)?[^@]*@(?P[\w.-]+?)\s*' diff --git a/utils/update_test_checks.py b/utils/update_test_checks.py index 014b55c9ae8..be15ae9e268 100755 --- a/utils/update_test_checks.py +++ b/utils/update_test_checks.py @@ -32,18 +32,12 @@ designed to be authoratitive about what constitutes a good test! from __future__ import print_function import argparse -import glob -import itertools -import os # Used to advertise this file's name ("autogenerated_note"). -import string -import subprocess -import sys -import tempfile +import os # Used to advertise this file's name ("autogenerated_note"). import re +import sys from UpdateTestChecks import common -ADVERT = '; NOTE: Assertions have been autogenerated by ' def main(): from argparse import RawTextHelpFormatter @@ -58,58 +52,26 @@ def main(): help='Keep function signature information around for the check line') parser.add_argument('--scrub-attributes', action='store_true', help='Remove attribute annotations (#0) from the end of check line') - parser.add_argument('--enable', action='store_true', dest='enabled', default=True, - help='Activate CHECK line generation from this point forward') - parser.add_argument('--disable', action='store_false', dest='enabled', - help='Deactivate CHECK line generation from this point forward') parser.add_argument('tests', nargs='+') - args = common.parse_commandline_args(parser) + initial_args = common.parse_commandline_args(parser) script_name = os.path.basename(__file__) - autogenerated_note = (ADVERT + 'utils/' + script_name) - - opt_basename = os.path.basename(args.opt_binary) + opt_basename = os.path.basename(initial_args.opt_binary) if not re.match(r'^opt(-\d+)?$', opt_basename): common.error('Unexpected opt name: ' + opt_basename) sys.exit(1) opt_basename = 'opt' - for test in args.tests: - if not glob.glob(test): - common.warn("Test file pattern '%s' was not found. Ignoring it." % (test,)) - continue - - # On Windows we must expand the patterns ourselves. - test_paths = [test for pattern in args.tests for test in glob.glob(pattern)] - for test in test_paths: - argv = sys.argv[:] - args = parser.parse_args() - with open(test) as f: - input_lines = [l.rstrip() for l in f] - - first_line = input_lines[0] if input_lines else "" - if 'autogenerated' in first_line and script_name not in first_line: - common.warn("Skipping test which wasn't autogenerated by " + script_name, test) - continue - if first_line and 'autogenerated' in first_line: - args, argv = common.check_for_command(first_line, parser, args, argv) - test_autogenerated_note = autogenerated_note + common.get_autogennote_suffix(parser, args) - - if args.update_only: - if not first_line or 'autogenerated' not in first_line: - common.warn("Skipping test which isn't autogenerated: " + test) - continue - - run_lines = common.find_run_lines(test, input_lines) - + for ti in common.itertests(initial_args.tests, parser, + script_name='utils/' + script_name): # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces - if args.scrub_attributes: + if ti.args.scrub_attributes: common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE else: common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_RE prefix_list = [] - for l in run_lines: + for l in ti.run_lines: if '|' not in l: common.warn('Skipping unparseable RUN line: ' + l) continue @@ -127,8 +89,9 @@ def main(): tool_cmd_args = tool_cmd[len(opt_basename):].strip() tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip() - check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd) - for item in m.group(1).split(',')] + check_prefixes = [item for m in + common.CHECK_PREFIX_RE.finditer(filecheck_cmd) + for item in m.group(1).split(',')] if not check_prefixes: check_prefixes = ['CHECK'] @@ -144,28 +107,20 @@ def main(): common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args) common.debug('Extracted FileCheck prefixes: ' + str(prefixes)) - raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test) + raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args, ti.path) common.build_function_body_dictionary( common.OPT_FUNCTION_RE, common.scrub_body, [], - raw_tool_output, prefixes, func_dict, args.verbose, - args.function_signature) + raw_tool_output, prefixes, func_dict, ti.args.verbose, + ti.args.function_signature) is_in_function = False is_in_function_start = False prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes]) common.debug('Rewriting FileCheck prefixes:', str(prefix_set)) output_lines = [] - output_lines.append(test_autogenerated_note) - - for input_line in input_lines: - # Discard any previous script advertising. - if input_line.startswith(ADVERT): - continue - - args, argv = common.check_for_command(input_line, parser, args, argv) - if not args.enabled: - output_lines.append(input_line) - continue + for input_line_info in ti.iterlines(output_lines): + input_line = input_line_info.line + args = input_line_info.args if is_in_function_start: if input_line == '': continue @@ -204,9 +159,9 @@ def main(): continue is_in_function = is_in_function_start = True - common.debug('Writing %d lines to %s...' % (len(output_lines), test)) + common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path)) - with open(test, 'wb') as f: + with open(ti.path, 'wb') as f: f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])