Changeset 136f86b for tests/test.py

Timestamp:

Mar 27, 2020, 11:45:09 AM (4 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

68887f9

Parents:

2980667

Message:

Test script can now run multiple configuration in a row

File:

• tests/test.py (modified) (9 diffs)

tests/test.py

@@ -6 +6 @@
 
 import argparse
+import itertools
 import re
 import sys
@@ -29 +30 @@
                         test.path = match.group(1)
                         test.arch = match.group(3)[1:] if match.group(3) else None
-                        if settings.arch.match(test.arch):
-                                expected.append(test)
+                        expected.append(test)
 
         path_walk( match_test )
@@ -52 +52 @@
                         x.target().startswith( tuple(excludes) )
                 ]
+
+        # sort the test alphabetically for convenience
+        test_list.sort(key=lambda t: ('~' if t.arch else '') + t.target() + (t.arch if t.arch else ''))
 
         return test_list
@@ -76 +79 @@
 
                         if test :
-                                tests.append( test[0] )
+                                tests.extend( test )
                         else :
                                 print('ERROR: No expected file for test %s, ignoring it' % testname, file=sys.stderr)
@@ -86 +89 @@
         # create a parser with the arguments for the tests script
         parser = argparse.ArgumentParser(description='Script which runs cforall tests')
-        parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='yes')
-        parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=yes_no, default='no')
-        parser.add_argument('--arch', help='Test for specific architecture', type=str, default='')
+        parser.add_argument('--debug', help='Run all tests in debug or release', type=comma_separated(yes_no), default='yes')
+        parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=comma_separated(yes_no), default='no')
+        parser.add_argument('--arch', help='Test for specific architecture', type=comma_separated(str), default='')
+        parser.add_argument('--continue', help='When multiple specifications are passed (debug/install/arch), sets whether or not to continue if the last specification failed', type=yes_no, default='yes', dest='continue_')
         parser.add_argument('--timeout', help='Maximum duration in seconds after a single test is considered to have timed out', type=int, default=60)
         parser.add_argument('--global-timeout', help='Maximum cumulative duration in seconds after the ALL tests are considered to have timed out', type=int, default=7200)
@@ -279 +283 @@
         make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
 
-        return 1 if failed else 0
+        return failed
 
 
@@ -298 +302 @@
                 tests = list_tests( None, None )
 
-                # sort the test alphabetically for convenience
-                tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
                 # print the possible options
                 print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs ", end='')
@@ -309 +310 @@
                 tests = list_tests( options.include, options.exclude )
 
-                # sort the test alphabetically for convenience
-                tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
                 # print the available tests
-                print("Listing for %s:%s"% (settings.arch.string, settings.debug.string))
                 fancy_print("\n".join(map(lambda t: t.toString(), tests)))
 
@@ -333 +330 @@
                         sys.exit(1)
 
-                # sort the test alphabetically for convenience
-                tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
-                # check the build configuration works
+                # prep invariants
                 settings.prep_output(tests)
-                settings.validate()
-
-                options.jobs, forceJobs = job_count( options, tests )
-                settings.update_make_cmd(forceJobs, options.jobs)
-
-                print('%s %i tests on %i cores (%s:%s)' % (
-                        'Regenerating' if settings.generating else 'Running',
-                        len(tests),
-                        options.jobs,
-                        settings.arch.string,
-                        settings.debug.string
-                ))
-
-                # otherwise run all tests and make sure to return the correct error code
-                sys.exit( run_tests(tests, options.jobs) )
+                failed = 0
+
+                # for each build configurations, run the test
+                for arch, debug, install in itertools.product(settings.all_arch, settings.all_debug, settings.all_install):
+                        settings.arch    = arch
+                        settings.debug   = debug
+                        settings.install = install
+
+                        # filter out the tests for a different architecture
+                        # tests are the same across debug/install
+                        local_tests = settings.arch.filter( tests )
+                        options.jobs, forceJobs = job_count( options, local_tests )
+                        settings.update_make_cmd(forceJobs, options.jobs)
+
+                        # check the build configuration works
+                        settings.validate()
+
+                        # print configuration
+                        print('%s %i tests on %i cores (%s:%s)' % (
+                                'Regenerating' if settings.generating else 'Running',
+                                len(local_tests),
+                                options.jobs,
+                                settings.arch.string,
+                                settings.debug.string
+                        ))
+
+                        # otherwise run all tests and make sure to return the correct error code
+                        failed = run_tests(local_tests, options.jobs)
+                        if failed:
+                                result = 1
+                                if not settings.continue_:
+                                        break
+
+
+                sys.exit( failed )