Context Navigation

-                      r9c59cd4
+                      rc07d724
 from os import listdir, environ
 from os.path import isfile, join, splitext
 from subprocess import Popen, PIPE, STDOUT
+from pybin.tools import *
 import argparse
 …
 import re
 import signal
-import stat
 import sys
 …
 def getMachineType():
         sh('echo "void ?{}(int*a,int b){}int main(){return 0;}" > .dummy.c')
         ret, out = sh("make .dummy -s", print2stdout=False)
+        ret, out = sh("make .dummy -s", print2stdout=True)
         if ret != 0:
 …
                 print(out)
                 print("Stopping")
+                sh("rm -f .dummy.c > /dev/null 2>&1")
+                sh("rm -f .dummy > /dev/null 2>&1")
+                rm( (".dummy.c",".dummy") )
                 sys.exit(1)
         _, out = sh("file .dummy", print2stdout=False)
         sh("rm -f .dummy.c > /dev/null 2>&1")
+        sh("rm -f .dummy > /dev/null 2>&1")
+        rm( (".dummy.c",".dummy") )
         return re.search("ELF\s([0-9]+)-bit", out).group(1)
 …
         return generic_list + typed_list + concurrent_list;
+# helper functions to run terminal commands
+def sh(cmd, dry_run = False, print2stdout = True):
+        if dry_run :    # if this is a dry_run, only print the commands that would be ran
+                print("cmd: %s" % cmd)
+                return 0, None
+        else :                  # otherwise create a pipe and run the desired command
+                proc = Popen(cmd, stdout=None if print2stdout else PIPE, stderr=STDOUT, shell=True)
+                out, err = proc.communicate()
+                return proc.returncode, out
+# helper function to replace patterns in a file
+def file_replace(fname, pat, s_after):
+    # first, see if the pattern is even in the file.
+    with open(fname) as f:
+        if not any(re.search(pat, line) for line in f):
+            return # pattern does not occur in file so we are done.
+    # pattern is in the file, so perform replace operation.
+    with open(fname) as f:
+        out_fname = fname + ".tmp"
+        out = open(out_fname, "w")
+        for line in f:
+            out.write(re.sub(pat, s_after, line))
+        out.close()
+        os.rename(out_fname, fname)
+# tests output may differ depending on the depth of the makefile
+def fix_MakeLevel(file) :
+        if environ.get('MAKELEVEL') :
+                file_replace(file, "make\[%i\]" % int(environ.get('MAKELEVEL')), 'make' )
+# helper function to check if a files contains only a spacific string
+def fileContainsOnly(file, text) :
+        with open(file) as f:
+                ff = f.read().strip()
+                result = ff == text.strip()
+                return result;
+# check whether or not a file is executable
+def fileIsExecutable(file) :
+        try :
+                fileinfo = os.stat(file)
+                return bool(fileinfo.st_mode & stat.S_IXUSR)
+        except Exception as inst:
+                print(type(inst))    # the exception instance
+                print(inst.args)     # arguments stored in .args
+                print(inst)
+                return False
+################################################################################
+#               running test functions
+################################################################################
+def run_single_test(test, generate, dry_run, debug):
+        try :
+                # find the output file based on the test name and options flag
+                out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path)
+                err_file = ".err/%s.log" % test.name
+                # remove any outputs from the previous tests to prevent side effects
+                sh("rm -f %s" % out_file, dry_run)
+                sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
+                options = "-debug" if debug else "-nodebug";
+                # build, skipping to next test on error
+                make_ret, _ = sh("""%s test=yes EXTRA_FLAGS="-quiet %s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)
+                # if the make command succeds continue otherwise skip to diff
+                if make_ret == 0 :
+                        # fetch optional input
+                        stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.path) else ""
+                        if fileIsExecutable(test.name) :
+                                # run test
+                                sh("./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run)
+                        else :
+                                # simply cat the result into the output
+                                sh("cat %s > %s" % (test.name, out_file), dry_run)
+                else :
+                        # command failed save the log to less temporary file
+                        sh("mv %s %s" % (err_file, out_file), dry_run)
+                retcode = 0
+                error = None
+                # # fix output to prevent make depth to cause issues
+                # fix_MakeLevel(out_file)
+                if generate :
+                        # if we are ounly generating the output we still need to check that the test actually exists
+                        if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.name) :
+                                retcode = 1;
+                                error = "\t\tNo make target for test %s!" % test.name
+                                sh("rm %s" % out_file, False)
+                else :
+                        # diff the output of the files
+                        diff_cmd = ("diff --ignore-all-space "
+                                                "--ignore-blank-lines "
+                                                "--old-group-format='\t\tmissing lines :\n"
+                                                "%%<' \\\n"
+                                                "--new-group-format='\t\tnew lines :\n"
+                                                "%%>' \\\n"
+                                                "--unchanged-group-format='%%=' \\"
+                                                "--changed-group-format='\t\texpected :\n"
+                                                "%%<\n"
+                                                "\t\tgot :\n"
+                                                "%%>' \\\n"
+                                                "--new-line-format='\t\t%%dn\t%%L' \\\n"
+                                                "--old-line-format='\t\t%%dn\t%%L' \\\n"
+                                                "--unchanged-line-format='' \\\n"
+                                                ".expect/%s.txt .out/%s.log")
+                        # fetch return code and error from the diff command
+                        retcode, error = sh(diff_cmd % (test.path, test.name), dry_run, False)
+        finally :
+                # clean the executable
+                sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
+        return retcode, error
+def run_test_instance(t, generate, dry_run, debug) :
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+        # print formated name
+        name_txt = "%20s  " % t.name
+        #run the test instance and collect the result
+        test_failed, error = run_single_test(t, generate, dry_run, debug)
+        # update output based on current action
+        if generate :
+                failed_txt = "ERROR"
+                success_txt = "Done"
+        else :
+                failed_txt = "FAILED"
+                success_txt = "PASSED"
+        #print result with error if needed
+        text = name_txt + (failed_txt if test_failed else success_txt)
+        out = sys.stdout
+        if error :
+                text = text + "\n" + error
+                out = sys.stderr
+        print(text, file = out);
+        sys.stdout.flush()
+        sys.stderr.flush()
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        return test_failed
+# run the given list of tests with the given parameters
+def run_tests(tests, generate, dry_run, jobs, debug) :
+        # clean the sandbox from previous commands
+        sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
+        #make sure the required folder are present
+        sh('mkdir -p .out .expect .err', dry_run)
+        if generate :
+                print( "Regenerate tests for: " )
+        # for each test to run
+        original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        pool = Pool(jobs)
+        signal.signal(signal.SIGINT, original_sigint_handler)
+        try :
+                results = pool.map_async(partial(run_test_instance, generate=generate, dry_run=dry_run, debug=debug), tests ).get(9999)
+        except KeyboardInterrupt:
+                pool.terminate()
+                print("Tests interrupted by user")
+                sys.exit(1)
+        #clean the workspace
+        sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
+        for failed in results:
+                if failed :
+                        return 1
+        return 0
+def yes_no(string):
+        if string == "yes" :
+                return True
+        if string == "no" :
+                return False
+        raise argparse.ArgumentTypeError(msg)
+        return False
+################################################################################
+#               main loop
+################################################################################
+abspath = os.path.abspath(__file__)
+dname = os.path.dirname(abspath)
+os.chdir(dname)
+# create a parser with the arguments for the tests script
+parser = argparse.ArgumentParser(description='Script which runs cforall tests')
+parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no')
+parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes')
+parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
+parser.add_argument('--list', help='List all test available', action='store_true')
+parser.add_argument('--all', help='Run all test available', action='store_true')
+parser.add_argument('--regenerate-expected', help='Regenerate the .expect by running the specified tets, can be used with --all option', action='store_true')
+parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8')
+parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
+parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
+# parse the command line arguments
+options = parser.parse_args()
+do_list = options.list or options.list_comp
+# script must have at least some tests to run
+if (len(options.tests) > 0  and     options.all and not do_list) \
+or (len(options.tests) == 0 and not options.all and not do_list) :
+        print('ERROR: must have option \'--all\' or non-empty test list', file=sys.stderr)
+        parser.print_help()
+        sys.exit(1)
+# fetch the liest of all valid tests
+allTests = listTests( options.concurrent )
+# if user wants all tests than no other treatement of the test list is required
+if options.all or do_list :
+        tests = allTests
+else :
+        #otherwise we need to validate that the test list that was entered is valid
+# from the found tests, filter all the valid tests/desired tests
+def validTests( options ):
         tests = []
 …
         if options.regenerate_expected :
                 for testname in options.tests :
                         if testname.endswith(".c") or testname.endswith(".cc") or testname.endswith(".cpp") :
+                        if testname.endswith( (".c", ".cc", ".cpp") ):
                                 print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr)
                         else :
 …
                 sys.exit(1)
+# sort the test alphabetically for convenience
+tests.sort(key=lambda t: t.name)
+# users may want to simply list the tests
+if options.list_comp :
+        print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')
+        print(" ".join(map(lambda t: "%s" % (t.name), tests)))
+elif options.list :
+        print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests)))
+else :
+        return tests
+# parses the option
+def getOptions():
+        # create a parser with the arguments for the tests script
+        parser = argparse.ArgumentParser(description='Script which runs cforall tests')
+        parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no')
+        parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes')
+        parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
+        parser.add_argument('--list', help='List all test available', action='store_true')
+        parser.add_argument('--all', help='Run all test available', action='store_true')
+        parser.add_argument('--regenerate-expected', help='Regenerate the .expect by running the specified tets, can be used with --all option', action='store_true')
+        parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8')
+        parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
+        parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
+        options =  parser.parse_args()
+        # script must have at least some tests to run or be listing
+        listing    = options.list or options.list_comp
+        all_tests  = options.all
+        some_tests = len(options.tests) > 0
+        # check that exactly one of the booleans is set to true
+        if not sum( (listing, all_tests, some_tests) ) == 1 :
+                print('ERROR: must have option \'--all\', \'--list\' or non-empty test list', file=sys.stderr)
+                parser.print_help()
+                sys.exit(1)
+        return options
+def jobCount( options ):
         # check if the user already passed in a number of jobs for multi-threading
         make_flags = environ.get('MAKEFLAGS')
 …
                 sys.exit(1)
+        options.jobs = min( options.jobs, len(tests) )
+        print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))
+        make_cmd = "make" if make_flags else ("make -j%i" % options.jobs)
+        # otherwise run all tests and make sure to return the correct error code
+        sys.exit( run_tests(tests, options.regenerate_expected, options.dry_run, options.jobs, options.debug) )
+        return min( options.jobs, len(tests) ), True if make_flags else False
+################################################################################
+#               running test functions
+################################################################################
+# logic to run a single test and return the result (No handling of printing or other test framework logic)
+def run_single_test(test, generate, dry_run, debug):
+        # find the output file based on the test name and options flag
+        out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path)
+        err_file = ".err/%s.log" % test.name
+        # remove any outputs from the previous tests to prevent side effects
+        rm( (out_file, test.name), dry_run )
+        options = "-debug" if debug else "-nodebug"
+        # build, skipping to next test on error
+        make_ret, _ = sh("""%s test=yes EXTRA_FLAGS="-quiet %s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)
+        # if the make command succeds continue otherwise skip to diff
+        if make_ret == 0 :
+                # fetch optional input
+                stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.path) else ""
+                if fileIsExecutable(test.name) :
+                        # run test
+                        sh("./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run)
+                else :
+                        # simply cat the result into the output
+                        sh("cat %s > %s" % (test.name, out_file), dry_run)
+        else :
+                # command failed save the log to less temporary file
+                sh("mv %s %s" % (err_file, out_file), dry_run)
+        retcode = 0
+        error = None
+        if generate :
+                # if we are ounly generating the output we still need to check that the test actually exists
+                if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.name) :
+                        retcode = 1;
+                        error = "\t\tNo make target for test %s!" % test.name
+                        sh("rm %s" % out_file, False)
+        else :
+                # fetch return code and error from the diff command
+                retcode, error = diff(".expect/%s.txt" % test.path, ".out/%s.log" % test.name, dry_run)
+        # clean the executable
+        sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
+        return retcode, error
+# run a single test and handle the errors, outputs, printing, exception handling, etc.
+def run_test_worker(t, generate, dry_run, debug) :
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+        # print formated name
+        name_txt = "%20s  " % t.name
+        #run the test instance and collect the result
+        test_failed, error = run_single_test(t, generate, dry_run, debug)
+        # update output based on current action
+        if generate :
+                failed_txt = "ERROR"
+                success_txt = "Done"
+        else :
+                failed_txt = "FAILED"
+                success_txt = "PASSED"
+        #print result with error if needed
+        text = name_txt + (failed_txt if test_failed else success_txt)
+        out = sys.stdout
+        if error :
+                text = text + "\n" + error
+                out = sys.stderr
+        print(text, file = out);
+        sys.stdout.flush()
+        sys.stderr.flush()
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        return test_failed
+# run the given list of tests with the given parameters
+def run_tests(tests, generate, dry_run, jobs, debug) :
+        # clean the sandbox from previous commands
+        sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
+        # make sure the required folder are present
+        sh('mkdir -p .out .expect .err', dry_run)
+        if generate :
+                print( "Regenerate tests for: " )
+        # create the executor for our jobs and handle the signal properly
+        original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        pool = Pool(jobs)
+        signal.signal(signal.SIGINT, original_sigint_handler)
+        # for each test to run
+        try :
+                results = pool.map_async(partial(run_test_worker, generate=generate, dry_run=dry_run, debug=debug), tests ).get(3600)
+        except KeyboardInterrupt:
+                pool.terminate()
+                print("Tests interrupted by user")
+                sys.exit(1)
+        # clean the workspace
+        sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
+        for failed in results:
+                if failed :
+                        return 1
+        return 0
+################################################################################
+#               main loop
+################################################################################
+if __name__ == "__main__":
+        #always run from same folder
+        chdir()
+        # parse the command line arguments
+        options = getOptions()
+        # fetch the liest of all valid tests
+        allTests = listTests( options.concurrent )
+        # if user wants all tests than no other treatement of the test list is required
+        if options.all or options.list or options.list_comp :
+                tests = allTests
+        else :
+                #otherwise we need to validate that the test list that was entered is valid
+                tests = validTests( options )
+        # sort the test alphabetically for convenience
+        tests.sort(key=lambda t: t.name)
+        # users may want to simply list the tests
+        if options.list_comp :
+                print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')
+                print(" ".join(map(lambda t: "%s" % (t.name), tests)))
+        elif options.list :
+                print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests)))
+        else :
+                options.jobs, forceJobs = jobCount( options )
+                print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))
+                make_cmd = "make" if forceJobs else ("make -j%i" % options.jobs)
+                # otherwise run all tests and make sure to return the correct error code
+                sys.exit( run_tests(tests, options.regenerate_expected, options.dry_run, options.jobs, options.debug) )

Note: See TracChangeset for help on using the changeset viewer.

Context Navigation

Changeset c07d724

Legend:

.gitignore

src/tests/test.py

Download in other formats: