Index: src/tests/test.py =================================================================== --- src/tests/test.py (revision ed5ad08f3e072560ac0cab42c471aedc291c22dd) +++ src/tests/test.py (revision 911348cd2ef4d7f3e629b98e229290ec3cb7b894) @@ -16,8 +16,10 @@ ################################################################################ +# Test class that defines what a test is class Test: def __init__(self, name, path): self.name, self.path = name, path +# parses the Makefile to find the machine type (32-bit / 64-bit) def getMachineType(): with open('Makefile') as file: @@ -26,7 +28,9 @@ return m.group(1) +# reads the directory ./.expect and indentifies the tests def listTests(): machineType = getMachineType() + # tests directly in the .expect folder will always be processed generic_list = map(lambda fname: Test(fname, fname), [splitext(f)[0] for f in listdir('./.expect') @@ -34,4 +38,5 @@ ]) + # tests in the machineType folder will be ran only for the corresponding compiler typed_list = map(lambda fname: Test( fname, "%s/%s" % (machineType, fname) ), [splitext(f)[0] for f in listdir("./.expect/%s" % machineType) @@ -39,15 +44,18 @@ ]) + # append both lists to get return generic_list + typed_list +# helper functions to run terminal commands def sh(cmd, dry_run = False, print2stdout = True): - if dry_run : + if dry_run : # if this is a dry_run, only print the commands that would be ran print("cmd: %s" % cmd) return 0, None - else : + else : # otherwise create a pipe and run the desired command proc = Popen(cmd, stdout=None if print2stdout else PIPE, stderr=STDOUT, shell=True) out, err = proc.communicate() return proc.returncode, out +# helper function to replace patterns in a file def file_replace(fname, pat, s_after): # first, see if the pattern is even in the file. @@ -65,19 +73,18 @@ os.rename(out_fname, fname) +# tests output may differ depending on the depth of the makefile def fix_MakeLevel(file) : if environ.get('MAKELEVEL') : file_replace(file, "make\[%i\]" % int(environ.get('MAKELEVEL')), 'make' ) +# helper function to check if a files contains only a spacific string def fileContainsOnly(file, text) : with open(file) as f: ff = f.read().strip() result = ff == text.strip() - # - # print("Comparing :\n\t'%s'\nWith:\n\t'%s'" % (ff, text)) - # print("Result is : \n\t", end="") - # print(result) return result; +# check whether or not a file is executable def fileIsExecutable(file) : try : @@ -90,4 +97,5 @@ return False +# find the test data for a given test name def filterTests(testname) : found = [test for test in allTests if test.name == testname] @@ -99,6 +107,8 @@ def run_test_instance(test, generate, dry_run): + # find the output file based on the test name and options flag out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path) + # remove any outputs from the previous tests to prevent side effects sh("rm -f %s" % out_file, dry_run) sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run) @@ -107,4 +117,5 @@ make_ret, _ = sh("%s %s 2> %s 1> /dev/null" % (make_cmd, test.name, out_file), dry_run) + # if the make command succeds continue otherwise skip to diff if make_ret == 0 : # fetch optional input @@ -121,7 +132,9 @@ error = None + # fix output to prevent make depth to cause issues fix_MakeLevel(out_file) if generate : + # if we are ounly generating the output we still need to check that the test actually exists if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'. Stop." % test.name) : retcode = 1; @@ -145,4 +158,5 @@ ".expect/%s.txt .out/%s.log") + # fetch return code and error from the diff command retcode, error = sh(diff_cmd % (test.path, test.name), dry_run, False) @@ -152,6 +166,10 @@ return retcode, error -def run_tests(tests, generate, dry_run) : +# run the given list of tests with the given parameters +def run_tests(tests, generate, dry_run, jobs) : + # clean the sandbox from previous commands sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run) + + #make sure the required folder are present sh('mkdir -p .out .expect', dry_run) @@ -160,10 +178,17 @@ failed = False; + # for eeach test to run for t in tests: + # print formated name print("%20s " % t.name, end="") sys.stdout.flush() + + #run the test instance and collect the result test_failed, error = run_test_instance(t, generate, dry_run) + + # aggregate test suite result failed = test_failed or failed + # update output based on current action if generate : failed_txt = "ERROR" @@ -173,8 +198,10 @@ success_txt = "PASSED" + #print result with error if needed print(failed_txt if test_failed else success_txt) if error : print(error, file=sys.stderr) + #clean the workspace sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run) @@ -184,4 +211,5 @@ # main loop ################################################################################ +# create a parser with the arguments for the tests script parser = argparse.ArgumentParser(description='Script which runs cforall tests') parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true') @@ -189,8 +217,11 @@ parser.add_argument('--all', help='Run all test available', action='store_true') parser.add_argument('--regenerate-expected', help='Regenerate the .expect by running the specified tets, can be used with --all option', action='store_true') +parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='1') parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run') +# parse the command line arguments options = parser.parse_args() +# script must have at least some tests to run if (len(options.tests) > 0 and options.all and not options.list) \ or (len(options.tests) == 0 and not options.all and not options.list) : @@ -199,16 +230,22 @@ sys.exit(1) +# fetch the liest of all valid tests allTests = listTests() +# if user wants all tests than no other treatement of the test list is required if options.all or options.list : tests = allTests else : + #otherwise we need to validate that the test list that was entered is valid tests = [] + # if we are regenerating the tests we need to find the information of the + # already existing tests and create new info for the new tests if options.regenerate_expected : tests = map(filterTests, options.tests) else : + # otherwise we only need to validate that all tests are present in the complete list for testname in options.tests: test = [t for t in allTests if t.name == testname] @@ -219,16 +256,30 @@ print('ERROR: No expected file for test %s, ignoring it' % testname, file=sys.stderr) + # make sure we have at least some test to run if len(tests) == 0 : print('ERROR: No valid test to run', file=sys.stderr) sys.exit(1) +# sort the test alphabetically for convenience tests.sort(key=lambda t: t.name) +# check if the user already passed in a number of jobs for multi-threading make_flags = environ.get('MAKEFLAGS') -make_cmd = "make" if make_flags and "-j" in make_flags else "make -j8" - +make_has_max_jobs = make_flags and "-j" in make_flags +make_max_jobs = re.search("(-j|--jobs)\s*([0-9]+)", make_flags).group(2) if make_has_max_jobs else None +make_cmd = "make" if make_has_max_jobs else "make -j8" + +# make sure we have a valid number of jobs that corresponds to user input +options.jobs = int(make_max_jobs) if make_max_jobs else options.jobs +if options.jobs <= 0 : + print('ERROR: Invalid number of jobs', file=sys.stderr) + sys.exit(1) + + +# users may want to simply list the tests if options.list : print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests))) else : + # otherwise run all tests and make sure to return the correct error code sys.exit( run_tests(tests, options.regenerate_expected, options.dry_run) )