Changeset 6a490b2 for tests/test.py

Timestamp:

May 11, 2020, 1:53:29 PM (6 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

504a7dc

Parents:

b7d6a36 (diff), a7b486b (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into relaxed_ready

File:

• tests/test.py (modified) (10 diffs)

tests/test.py

@@ -6 +6 @@
 
 import argparse
+import itertools
 import re
 import sys
@@ -29 +30 @@
                         test.path = match.group(1)
                         test.arch = match.group(3)[1:] if match.group(3) else None
-                        if settings.arch.match(test.arch):
-                                expected.append(test)
+                        expected.append(test)
 
         path_walk( match_test )
@@ -53 +53 @@
                 ]
 
+        # sort the test alphabetically for convenience
+        test_list.sort(key=lambda t: ('~' if t.arch else '') + t.target() + (t.arch if t.arch else ''))
+
         return test_list
 
@@ -64 +67 @@
                 for testname in options.tests :
                         testname = canonical_path( testname )
+                        # first check if this is a valid name to regenerate
                         if Test.valid_name(testname):
+                                # this is a valid name, let's check if it already exists
                                 found = [test for test in all_tests if canonical_path( test.target() ) == testname]
-                                tests.append( found[0] if len(found) == 1 else Test.from_target(testname) )
+                                if not found:
+                                        # it's a new name, create it according to the name and specified architecture
+                                        if options.arch:
+                                                # user specified one or multiple architectures, assume the tests will have architecture specific results
+                                                tests.extend( [Test.new_target(testname, arch) for arch in settings.all_arch] )
+                                        else:
+                                                # user didn't specify an architecture, just create a cross platform test
+                                                tests.append( Test.new_target( testname, None ) )
+                                elif len(found) == 1 and not found[0].arch:
+                                        # we found a single test, the user better be wanting to create a cross platform test
+                                        if options.arch:
+                                                print('ERROR: "%s", test has no specified architecture but --arch was specified, ignoring it' % testname, file=sys.stderr)
+                                        else:
+                                                tests.append( found[0] )
+                                else:
+                                        # this test is already cross platform, just add a test for each platform the user asked
+                                        tests.extend( [Test.new_target(testname, arch) for arch in settings.all_arch] )
+
+                                        # print a warning if it users didn't ask for a specific architecture
+                                        if not options.arch:
+                                                print('WARNING: "%s", test has architecture specific expected files but --arch was not specified, regenerating only for current host' % testname, file=sys.stderr)
+
                         else :
                                 print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr)
@@ -76 +102 @@
 
                         if test :
-                                tests.append( test[0] )
+                                tests.extend( test )
                         else :
                                 print('ERROR: No expected file for test %s, ignoring it' % testname, file=sys.stderr)
@@ -86 +112 @@
         # create a parser with the arguments for the tests script
         parser = argparse.ArgumentParser(description='Script which runs cforall tests')
-        parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='yes')
-        parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=yes_no, default='no')
-        parser.add_argument('--arch', help='Test for specific architecture', type=str, default='')
+        parser.add_argument('--debug', help='Run all tests in debug or release', type=comma_separated(yes_no), default='yes')
+        parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=comma_separated(yes_no), default='no')
+        parser.add_argument('--arch', help='Test for specific architecture', type=comma_separated(str), default=None)
+        parser.add_argument('--continue', help='When multiple specifications are passed (debug/install/arch), sets whether or not to continue if the last specification failed', type=yes_no, default='yes', dest='continue_')
         parser.add_argument('--timeout', help='Maximum duration in seconds after a single test is considered to have timed out', type=int, default=60)
         parser.add_argument('--global-timeout', help='Maximum cumulative duration in seconds after the ALL tests are considered to have timed out', type=int, default=7200)
+        parser.add_argument('--timeout-with-gdb', help='Instead of killing the command when it times out, orphan it and print process id to allow gdb to attach', type=yes_no, default="no")
         parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
         parser.add_argument('--list', help='List all test available', action='store_true')
@@ -178 +206 @@
 
                 else:
-                        with open (out_file, "r") as myfile:
-                                error = myfile.read()
+                        if os.stat(out_file).st_size < 1048576:
+                                with open (out_file, "r") as myfile:
+                                        error = myfile.read()
+                        else:
+                                error = "Output log can't be read, file is bigger than 1MB, see {} for actual error\n".format(out_file)
 
                         ret, info = core_info(exe_file)
@@ -215 +246 @@
                 return False, ""
         except Exception as ex:
-                print("Unexpected error in worker thread: %s" % ex, file=sys.stderr)
+                print("Unexpected error in worker thread running {}: {}".format(t.target(), ex), file=sys.stderr)
                 sys.stderr.flush()
                 return False, ""
@@ -278 +309 @@
         make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
 
-        return 1 if failed else 0
+        return failed
 
 
@@ -292 +323 @@
         settings.init( options )
 
-        # fetch the liest of all valid tests
-        all_tests = list_tests( options.include, options.exclude )
-
-
-        # if user wants all tests than no other treatement of the test list is required
-        if options.all or options.list or options.list_comp or options.include :
-                tests = all_tests
-
-        #otherwise we need to validate that the test list that was entered is valid
-        else :
-                tests = valid_tests( options )
-
-        # make sure we have at least some test to run
-        if not tests :
-                print('ERROR: No valid test to run', file=sys.stderr)
-                sys.exit(1)
-
-
-        # sort the test alphabetically for convenience
-        tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
         # users may want to simply list the tests
         if options.list_comp :
-                print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout -j --jobs ", end='')
+                # fetch the liest of all valid tests
+                tests = list_tests( None, None )
+
+                # print the possible options
+                print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs -I --include -E --exclude --continue ", end='')
                 print(" ".join(map(lambda t: "%s" % (t.target()), tests)))
 
         elif options.list :
-                print("Listing for %s:%s"% (settings.arch.string, settings.debug.string))
+                # fetch the liest of all valid tests
+                tests = list_tests( options.include, options.exclude )
+
+                # print the available tests
                 fancy_print("\n".join(map(lambda t: t.toString(), tests)))
 
         else :
-                # check the build configuration works
+                # fetch the liest of all valid tests
+                all_tests = list_tests( options.include, options.exclude )
+
+                # if user wants all tests than no other treatement of the test list is required
+                if options.all or options.include :
+                        tests = all_tests
+
+                #otherwise we need to validate that the test list that was entered is valid
+                else :
+                        tests = valid_tests( options )
+
+                # make sure we have at least some test to run
+                if not tests :
+                        print('ERROR: No valid test to run', file=sys.stderr)
+                        sys.exit(1)
+
+                # prep invariants
                 settings.prep_output(tests)
-                settings.validate()
-
-                options.jobs, forceJobs = job_count( options, tests )
-                settings.update_make_cmd(forceJobs, options.jobs)
-
-                print('%s %i tests on %i cores (%s:%s)' % (
-                        'Regenerating' if settings.generating else 'Running',
-                        len(tests),
-                        options.jobs,
-                        settings.arch.string,
-                        settings.debug.string
-                ))
-
-                # otherwise run all tests and make sure to return the correct error code
-                sys.exit( run_tests(tests, options.jobs) )
+                failed = 0
+
+                # for each build configurations, run the test
+                for arch, debug, install in itertools.product(settings.all_arch, settings.all_debug, settings.all_install):
+                        settings.arch    = arch
+                        settings.debug   = debug
+                        settings.install = install
+
+                        # filter out the tests for a different architecture
+                        # tests are the same across debug/install
+                        local_tests = settings.arch.filter( tests )
+                        options.jobs, forceJobs = job_count( options, local_tests )
+                        settings.update_make_cmd(forceJobs, options.jobs)
+
+                        # check the build configuration works
+                        settings.validate()
+
+                        # print configuration
+                        print('%s %i tests on %i cores (%s:%s)' % (
+                                'Regenerating' if settings.generating else 'Running',
+                                len(local_tests),
+                                options.jobs,
+                                settings.arch.string,
+                                settings.debug.string
+                        ))
+
+                        # otherwise run all tests and make sure to return the correct error code
+                        failed = run_tests(local_tests, options.jobs)
+                        if failed:
+                                result = 1
+                                if not settings.continue_:
+                                        break
+
+
+                sys.exit( failed )