Changeset 0ad0c55 for src/tests/test.py

Timestamp:

Dec 4, 2017, 1:19:42 PM (9 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, aaron-thesis, arm-eh, ast-experimental, cleanup-dtors, deferred_resn, demangler, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, new-env, no_list, persistent-indexer, pthread-emulation, qualifiedEnum, resolv-new, stuck-waitfor-destruct, with_gc

Children:

bacc36c

Parents:

b0e5593

Message:

Updated tests script to handle folders

File:

• src/tests/test.py (modified) (13 diffs)

src/tests/test.py

@@ -7 +7 @@
 from os.path import isfile, join, splitext
 from pybin.tools import *
+from pybin.test_run import *
 
 import argparse
@@ -19 +20 @@
 ################################################################################
 
-# Test class that defines what a test is
-class Test:
-    def __init__(self, name, path):
-        self.name, self.path = name, path
-
-class TestResult:
-        SUCCESS = 0
-        FAILURE = 1
-        TIMEOUT = 124
-
-# parses the Makefile to find the machine type (32-bit / 64-bit)
-def getMachineType():
-        sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c')
-        ret, out = sh("make .dummy -s", print2stdout=True)
-
-        if ret != 0:
-                print("Failed to identify architecture:")
-                print(out)
-                print("Stopping")
-                rm( (".dummy.c",".dummy") )
-                sys.exit(1)
-
-        _, out = sh("file .dummy", print2stdout=False)
-        rm( (".dummy.c",".dummy") )
-
-        return re.search("ELF\s([0-9]+)-bit", out).group(1)
-
-def listTestsFolder(folder) :
-        path = ('./.expect/%s/' % folder) if folder else './.expect/'
-        subpath = "%s/" % folder if folder else ""
+def list_expected():
+        expected = []
+
+        def step(_, dirname, names):
+                for name in names:
+                        path = os.path.join(dirname, name)
+
+                        match = re.search("(\.[\w\/\-_]*)\/.expect\/([\w\-_]+)(\.[\w\-_]+)?\.txt", path)
+                        if match :
+                                test = Test()
+                                test.name = match.group(2)
+                                test.path = match.group(1)
+                                test.arch = match.group(3)[1:] if match.group(3) else None
+                                expected.append(test)
+
+        # Start the walk
+        os.path.walk('.', step, '')
+
+        return expected
+
+# reads the directory ./.expect and indentifies the tests
+def listTests( includes, excludes ):
+        includes = [os.path.normpath( os.path.join('.',i) ) for i in includes] if includes else None
+        excludes = [os.path.normpath( os.path.join('.',i) ) for i in excludes] if excludes else None
 
         # tests directly in the .expect folder will always be processed
-        return map(lambda fname: Test(fname, subpath + fname),
-                [splitext(f)[0] for f in listdir( path )
-                if not f.startswith('.') and f.endswith('.txt')
-                ])
-
-# reads the directory ./.expect and indentifies the tests
-def listTests( concurrent ):
-        machineType = getMachineType()
-
-        # tests directly in the .expect folder will always be processed
-        generic_list = listTestsFolder( "" )
-
-        # tests in the machineType folder will be ran only for the corresponding compiler
-        typed_list = listTestsFolder( machineType )
-
-        # tests in the concurrent folder will be ran only if concurrency is enabled
-        concurrent_list = listTestsFolder( "concurrent" ) if concurrent else []
-
-        # append both lists to get
-        return generic_list + typed_list + concurrent_list;
+        test_list = list_expected()
+
+        # if we have a limited number of includes, filter by them
+        if includes:
+                test_list = [x for x in test_list if
+                        os.path.normpath( x.path ).startswith( tuple(includes) )
+                ]
+
+        # # if we have a folders to excludes, filter by them
+        if excludes:
+                test_list = [x for x in test_list if not
+                        os.path.normpath( x.path ).startswith( tuple(excludes) )
+                ]
+
+        return test_list
 
 # from the found tests, filter all the valid tests/desired tests
@@ -89 +79 @@
                 # otherwise we only need to validate that all tests are present in the complete list
                 for testname in options.tests:
-                        test = [t for t in allTests if t.name == testname]
+                        test = [t for t in allTests if os.path.normpath( t.target() ) == os.path.normpath( testname )]
 
                         if len(test) != 0 :
@@ -102 +92 @@
 
         return tests
+
+class TestResult:
+        SUCCESS = 0
+        FAILURE = 1
+        TIMEOUT = 124
 
 # parses the option
@@ -108 +103 @@
         parser = argparse.ArgumentParser(description='Script which runs cforall tests')
         parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no')
-        parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes')
+        parser.add_argument('--arch', help='Test for specific architecture', type=str, default=getMachineType())
         parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
         parser.add_argument('--list', help='List all test available', action='store_true')
@@ -115 +110 @@
         parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8')
         parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
+        parser.add_argument('-I','--include', help='Directory of test to include, can be used multiple time, All  if omitted', action='append')
+        parser.add_argument('-E','--exclude', help='Directory of test to exclude, can be used multiple time, None if omitted', action='append')
         parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
 
@@ -123 +120 @@
         all_tests  = options.all
         some_tests = len(options.tests) > 0
+        some_dirs  = len(options.include) > 0 if options.include else 0
 
         # check that exactly one of the booleans is set to true
-        if not sum( (listing, all_tests, some_tests) ) == 1 :
-                print('ERROR: must have option \'--all\', \'--list\' or non-empty test list', file=sys.stderr)
+        if not sum( (listing, all_tests, some_tests, some_dirs) ) > 0 :
+                print('ERROR: must have option \'--all\', \'--list\', \'--include\', \'-I\' or non-empty test list', file=sys.stderr)
                 parser.print_help()
                 sys.exit(1)
@@ -157 +155 @@
 
         # find the output file based on the test name and options flag
-        out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path)
-        err_file = ".err/%s.log" % test.name
+        out_file = test.output_file() if not generate else test.expect_file()
+        err_file = test.error_file()
+        cmp_file = test.expect_file()
+        in_file  = test.input_file()
+
+        # prepare the proper directories
+        test.prepare( dry_run )
 
         # remove any outputs from the previous tests to prevent side effects
-        rm( (out_file, err_file, test.name), dry_run )
+        rm( (out_file, err_file, test.target()), dry_run )
 
         options = "-debug" if debug else "-nodebug"
 
+
         # build, skipping to next test on error
-        make_ret, _ = sh("""%s test=yes DEBUG_FLAGS="%s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)
+        make_ret, _ = sh("""%s  DEBUG_FLAGS="%s" %s test="%s" 2> %s 1> /dev/null""" % (make_cmd, options, test.target(), err_file, out_file), dry_run)
 
         retcode = 0
@@ -172 +176 @@
 
         # if the make command succeds continue otherwise skip to diff
-        if make_ret == 0 :
+        if make_ret == 0 or dry_run:
                 # fetch optional input
-                stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.name) else ""
-
-                if fileIsExecutable(test.name) :
+                stdinput = "< %s" % in_file if isfile(in_file) else ""
+
+                if dry_run or fileIsExecutable(test.target()) :
                         # run test
-                        retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run)
+                        retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test.target(), stdinput, out_file), dry_run)
                 else :
                         # simply cat the result into the output
-                        sh("cat %s > %s" % (test.name, out_file), dry_run)
-
-        else :
-                # command failed save the log to less temporary file
+                        sh("cat %s > %s" % (test.target(), out_file), dry_run)
+        else:
                 sh("mv %s %s" % (err_file, out_file), dry_run)
+
 
         if retcode == 0:
                 if generate :
                         # if we are ounly generating the output we still need to check that the test actually exists
-                        if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.name) :
+                        if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.target()) :
                                 retcode = 1;
-                                error = "\t\tNo make target for test %s!" % test.name
+                                error = "\t\tNo make target for test %s!" % test.target()
                                 sh("rm %s" % out_file, False)
                 else :
                         # fetch return code and error from the diff command
-                        retcode, error = diff(".expect/%s.txt" % test.path, ".out/%s.log" % test.name, dry_run)
+                        retcode, error = diff(cmp_file, out_file, dry_run)
 
         else:
@@ -204 +207 @@
 
         # clean the executable
-        sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
+        sh("rm -f %s > /dev/null 2>&1" % test.target(), dry_run)
 
         return retcode, error
@@ -245 +248 @@
         # clean the sandbox from previous commands
         sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
-
-        # make sure the required folder are present
-        sh('mkdir -p .out .expect .err', dry_run)
 
         if generate :
@@ -286 +286 @@
 
         # fetch the liest of all valid tests
-        allTests = listTests( options.concurrent )
+        allTests = listTests( options.include, options.exclude )
 
         # if user wants all tests than no other treatement of the test list is required
-        if options.all or options.list or options.list_comp :
+        if options.all or options.list or options.list_comp or options.include :
                 tests = allTests
 
@@ -297 +297 @@
 
         # sort the test alphabetically for convenience
-        tests.sort(key=lambda t: t.name)
+        tests.sort(key=lambda t: os.path.join(t.path, t.name))
 
         # users may want to simply list the tests
         if options.list_comp :
-                print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')
-                print(" ".join(map(lambda t: "%s" % (t.name), tests)))
+                print("-h --help --debug --dry-run --list --all --regenerate-expected -j --jobs ", end='')
+                print(" ".join(map(lambda t: "%s" % (t.target()), tests)))
 
         elif options.list :
-                print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests)))
+                print("Listing for %s:%s"% (options.arch, "debug" if options.debug else "no debug"))
+                print("\n".join(map(lambda t: "%s" % (t.toString()), tests)))
 
         else :
                 options.jobs, forceJobs = jobCount( options )
 
-                print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))
+                print('Running (%s:%s) on %i cores' % (options.arch, "debug" if options.debug else "no debug", options.jobs))
                 make_cmd = "make" if forceJobs else ("make -j%i" % options.jobs)