Changes in tests/test.py [8364209:0f3d844]

File:

• tests/test.py (modified) (9 diffs)

tests/test.py

@@ -6 +6 @@
 
 import argparse
+import itertools
 import re
 import sys
 import tempfile
 import time
+
+import os
+import psutil
+import signal
 
 ################################################################################
@@ -25 +30 @@
                         test.path = match.group(1)
                         test.arch = match.group(3)[1:] if match.group(3) else None
-                        if settings.arch.match(test.arch):
-                                expected.append(test)
+                        expected.append(test)
 
         path_walk( match_test )
@@ -48 +52 @@
                         x.target().startswith( tuple(excludes) )
                 ]
+
+        # sort the test alphabetically for convenience
+        test_list.sort(key=lambda t: ('~' if t.arch else '') + t.target() + (t.arch if t.arch else ''))
 
         return test_list
@@ -72 +79 @@
 
                         if test :
-                                tests.append( test[0] )
+                                tests.extend( test )
                         else :
                                 print('ERROR: No expected file for test %s, ignoring it' % testname, file=sys.stderr)
@@ -82 +89 @@
         # create a parser with the arguments for the tests script
         parser = argparse.ArgumentParser(description='Script which runs cforall tests')
-        parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='yes')
-        parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=yes_no, default='no')
-        parser.add_argument('--arch', help='Test for specific architecture', type=str, default='')
+        parser.add_argument('--debug', help='Run all tests in debug or release', type=comma_separated(yes_no), default='yes')
+        parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=comma_separated(yes_no), default='no')
+        parser.add_argument('--arch', help='Test for specific architecture', type=comma_separated(str), default='')
+        parser.add_argument('--continue', help='When multiple specifications are passed (debug/install/arch), sets whether or not to continue if the last specification failed', type=yes_no, default='yes', dest='continue_')
         parser.add_argument('--timeout', help='Maximum duration in seconds after a single test is considered to have timed out', type=int, default=60)
         parser.add_argument('--global-timeout', help='Maximum cumulative duration in seconds after the ALL tests are considered to have timed out', type=int, default=7200)
+        parser.add_argument('--timeout-with-gdb', help='Instead of killing the command when it times out, orphan it and print process id to allow gdb to attach', type=yes_no, default="no")
         parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
         parser.add_argument('--list', help='List all test available', action='store_true')
@@ -148 +157 @@
         # run everything in a temp directory to make sure core file are handled properly
         with tempdir():
-                # if the make command succeds continue otherwise skip to diff
+                # if the make command succeeds continue otherwise skip to diff
                 if success(make_ret):
                         with Timed() as run_dur:
@@ -221 +230 @@
         make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
 
+        # since python prints stacks by default on a interrupt, redo the interrupt handling to be silent
+        def worker_init():
+                def sig_int(signal_num, frame):
+                        pass
+
+                signal.signal(signal.SIGINT, sig_int)
+
         # create the executor for our jobs and handle the signal properly
-        pool = multiprocessing.Pool(jobs)
+        pool = multiprocessing.Pool(jobs, worker_init)
 
         failed = False
+
+        def stop(x, y):
+                print("Tests interrupted by user", file=sys.stderr)
+                sys.exit(1)
+        signal.signal(signal.SIGINT, stop)
 
         # for each test to run
@@ -262 +283 @@
         make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
 
-        return 1 if failed else 0
+        return failed
 
 
@@ -276 +297 @@
         settings.init( options )
 
-        # fetch the liest of all valid tests
-        all_tests = list_tests( options.include, options.exclude )
-
-
-        # if user wants all tests than no other treatement of the test list is required
-        if options.all or options.list or options.list_comp or options.include :
-                tests = all_tests
-
-        #otherwise we need to validate that the test list that was entered is valid
-        else :
-                tests = valid_tests( options )
-
-        # make sure we have at least some test to run
-        if not tests :
-                print('ERROR: No valid test to run', file=sys.stderr)
-                sys.exit(1)
-
-
-        # sort the test alphabetically for convenience
-        tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
         # users may want to simply list the tests
         if options.list_comp :
-                print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout -j --jobs ", end='')
+                # fetch the liest of all valid tests
+                tests = list_tests( None, None )
+
+                # print the possible options
+                print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs -I --include -E --exclude --continue ", end='')
                 print(" ".join(map(lambda t: "%s" % (t.target()), tests)))
 
         elif options.list :
-                print("Listing for %s:%s"% (settings.arch.string, settings.debug.string))
+                # fetch the liest of all valid tests
+                tests = list_tests( options.include, options.exclude )
+
+                # print the available tests
                 fancy_print("\n".join(map(lambda t: t.toString(), tests)))
 
         else :
-                # check the build configuration works
+                # fetch the liest of all valid tests
+                all_tests = list_tests( options.include, options.exclude )
+
+                # if user wants all tests than no other treatement of the test list is required
+                if options.all or options.include :
+                        tests = all_tests
+
+                #otherwise we need to validate that the test list that was entered is valid
+                else :
+                        tests = valid_tests( options )
+
+                # make sure we have at least some test to run
+                if not tests :
+                        print('ERROR: No valid test to run', file=sys.stderr)
+                        sys.exit(1)
+
+                # prep invariants
                 settings.prep_output(tests)
-                settings.validate()
-
-                options.jobs, forceJobs = job_count( options, tests )
-                settings.update_make_cmd(forceJobs, options.jobs)
-
-                print('%s %i tests on %i cores (%s:%s)' % (
-                        'Regenerating' if settings.generating else 'Running',
-                        len(tests),
-                        options.jobs,
-                        settings.arch.string,
-                        settings.debug.string
-                ))
-
-                # otherwise run all tests and make sure to return the correct error code
-                sys.exit( run_tests(tests, options.jobs) )
+                failed = 0
+
+                # for each build configurations, run the test
+                for arch, debug, install in itertools.product(settings.all_arch, settings.all_debug, settings.all_install):
+                        settings.arch    = arch
+                        settings.debug   = debug
+                        settings.install = install
+
+                        # filter out the tests for a different architecture
+                        # tests are the same across debug/install
+                        local_tests = settings.arch.filter( tests )
+                        options.jobs, forceJobs = job_count( options, local_tests )
+                        settings.update_make_cmd(forceJobs, options.jobs)
+
+                        # check the build configuration works
+                        settings.validate()
+
+                        # print configuration
+                        print('%s %i tests on %i cores (%s:%s)' % (
+                                'Regenerating' if settings.generating else 'Running',
+                                len(local_tests),
+                                options.jobs,
+                                settings.arch.string,
+                                settings.debug.string
+                        ))
+
+                        # otherwise run all tests and make sure to return the correct error code
+                        failed = run_tests(local_tests, options.jobs)
+                        if failed:
+                                result = 1
+                                if not settings.continue_:
+                                        break
+
+
+                sys.exit( failed )