Changeset 136f86b


Ignore:
Timestamp:
Mar 27, 2020, 11:45:09 AM (18 months ago)
Author:
Thierry Delisle <tdelisle@…>
Branches:
arm-eh, jacob/cs343-translation, master, new-ast, new-ast-unique-expr
Children:
68887f9
Parents:
2980667
Message:

Test script can now run multiple configuration in a row

Location:
tests
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • tests/pybin/settings.py

    r2980667 r136f86b  
    7777                        print("updated to %s" % self.target)
    7878
    79         def match(self, arch):
     79        def filter(self, tests):
     80                return [test for test in tests if not test.arch or self.target == test.arch]
    8081                return True if not arch else self.target == arch
    8182
     
    113114
    114115def init( options ):
     116        global all_arch
     117        global all_debug
     118        global all_install
    115119        global arch
    116120        global archive
     121        global continue_
    117122        global debug
    118         global distcc
    119123        global dry_run
    120124        global generating
     
    125129        global timeout2gdb
    126130
    127         arch         = Architecture(options.arch)
     131        all_arch     = [Architecture(o) for o in list(dict.fromkeys(options.arch   ))]
     132        all_debug    = [Debug(o)        for o in list(dict.fromkeys(options.debug  ))]
     133        all_install  = [Install(o)      for o in list(dict.fromkeys(options.install))]
    128134        archive      = os.path.abspath(os.path.join(original_path, options.archive_errors)) if options.archive_errors else None
    129         debug        = Debug(options.debug)
     135        continue_    = options.continue_
    130136        dry_run      = options.dry_run # must be called before tools.config_hash()
    131         distcc       = "DISTCC_CFA_PATH=~/.cfadistcc/%s/cfa" % tools.config_hash()
    132137        generating   = options.regenerate_expected
    133         install      = Install(options.install)
    134138        make         = ['make']
    135139        output_width = 24
     
    148152
    149153def validate():
     154        """Validate the current configuration and update globals"""
     155
     156        global distcc
     157        distcc       = "DISTCC_CFA_PATH=~/.cfadistcc/%s/cfa" % tools.config_hash()
    150158        errf = os.path.join(BUILDDIR, ".validate.err")
    151159        make_ret, out = tools.make( ".validate", error_file = errf, output_file=subprocess.DEVNULL, error=subprocess.DEVNULL )
  • tests/pybin/tools.py

    r2980667 r136f86b  
    327327        raise argparse.ArgumentTypeError(msg)
    328328
     329# Convert a function that converts a string to one that converts comma separated string.
     330def comma_separated(elements):
     331    return lambda string: [elements(part) for part in string.split(',')]
     332
    329333def fancy_print(text):
    330334        column = which('column')
  • tests/test.py

    r2980667 r136f86b  
    66
    77import argparse
     8import itertools
    89import re
    910import sys
     
    2930                        test.path = match.group(1)
    3031                        test.arch = match.group(3)[1:] if match.group(3) else None
    31                         if settings.arch.match(test.arch):
    32                                 expected.append(test)
     32                        expected.append(test)
    3333
    3434        path_walk( match_test )
     
    5252                        x.target().startswith( tuple(excludes) )
    5353                ]
     54
     55        # sort the test alphabetically for convenience
     56        test_list.sort(key=lambda t: ('~' if t.arch else '') + t.target() + (t.arch if t.arch else ''))
    5457
    5558        return test_list
     
    7679
    7780                        if test :
    78                                 tests.append( test[0] )
     81                                tests.extend( test )
    7982                        else :
    8083                                print('ERROR: No expected file for test %s, ignoring it' % testname, file=sys.stderr)
     
    8689        # create a parser with the arguments for the tests script
    8790        parser = argparse.ArgumentParser(description='Script which runs cforall tests')
    88         parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='yes')
    89         parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=yes_no, default='no')
    90         parser.add_argument('--arch', help='Test for specific architecture', type=str, default='')
     91        parser.add_argument('--debug', help='Run all tests in debug or release', type=comma_separated(yes_no), default='yes')
     92        parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=comma_separated(yes_no), default='no')
     93        parser.add_argument('--arch', help='Test for specific architecture', type=comma_separated(str), default='')
     94        parser.add_argument('--continue', help='When multiple specifications are passed (debug/install/arch), sets whether or not to continue if the last specification failed', type=yes_no, default='yes', dest='continue_')
    9195        parser.add_argument('--timeout', help='Maximum duration in seconds after a single test is considered to have timed out', type=int, default=60)
    9296        parser.add_argument('--global-timeout', help='Maximum cumulative duration in seconds after the ALL tests are considered to have timed out', type=int, default=7200)
     
    279283        make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
    280284
    281         return 1 if failed else 0
     285        return failed
    282286
    283287
     
    298302                tests = list_tests( None, None )
    299303
    300                 # sort the test alphabetically for convenience
    301                 tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
    302 
    303304                # print the possible options
    304305                print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs ", end='')
     
    309310                tests = list_tests( options.include, options.exclude )
    310311
    311                 # sort the test alphabetically for convenience
    312                 tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
    313 
    314312                # print the available tests
    315                 print("Listing for %s:%s"% (settings.arch.string, settings.debug.string))
    316313                fancy_print("\n".join(map(lambda t: t.toString(), tests)))
    317314
     
    333330                        sys.exit(1)
    334331
    335                 # sort the test alphabetically for convenience
    336                 tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
    337 
    338                 # check the build configuration works
     332                # prep invariants
    339333                settings.prep_output(tests)
    340                 settings.validate()
    341 
    342                 options.jobs, forceJobs = job_count( options, tests )
    343                 settings.update_make_cmd(forceJobs, options.jobs)
    344 
    345                 print('%s %i tests on %i cores (%s:%s)' % (
    346                         'Regenerating' if settings.generating else 'Running',
    347                         len(tests),
    348                         options.jobs,
    349                         settings.arch.string,
    350                         settings.debug.string
    351                 ))
    352 
    353                 # otherwise run all tests and make sure to return the correct error code
    354                 sys.exit( run_tests(tests, options.jobs) )
     334                failed = 0
     335
     336                # for each build configurations, run the test
     337                for arch, debug, install in itertools.product(settings.all_arch, settings.all_debug, settings.all_install):
     338                        settings.arch    = arch
     339                        settings.debug   = debug
     340                        settings.install = install
     341
     342                        # filter out the tests for a different architecture
     343                        # tests are the same across debug/install
     344                        local_tests = settings.arch.filter( tests )
     345                        options.jobs, forceJobs = job_count( options, local_tests )
     346                        settings.update_make_cmd(forceJobs, options.jobs)
     347
     348                        # check the build configuration works
     349                        settings.validate()
     350
     351                        # print configuration
     352                        print('%s %i tests on %i cores (%s:%s)' % (
     353                                'Regenerating' if settings.generating else 'Running',
     354                                len(local_tests),
     355                                options.jobs,
     356                                settings.arch.string,
     357                                settings.debug.string
     358                        ))
     359
     360                        # otherwise run all tests and make sure to return the correct error code
     361                        failed = run_tests(local_tests, options.jobs)
     362                        if failed:
     363                                result = 1
     364                                if not settings.continue_:
     365                                        break
     366
     367
     368                sys.exit( failed )
Note: See TracChangeset for help on using the changeset viewer.