Changes in src/tests/test.py [f3b9efc:3eab308c]
- File:
-
- 1 edited
-
src/tests/test.py (modified) (11 diffs)
Legend:
- Unmodified
- Added
- Removed
-
src/tests/test.py
rf3b9efc r3eab308c 2 2 from __future__ import print_function 3 3 4 from functools import partial 5 from multiprocessing import Pool 6 from os import listdir, environ 7 from os.path import isfile, join, splitext 4 8 from pybin.tools import * 5 from pybin.test_run import *6 from pybin import settings7 9 8 10 import argparse 11 import multiprocessing 12 import os 9 13 import re 14 import signal 10 15 import sys 11 16 … … 14 19 ################################################################################ 15 20 16 def findTests(): 17 expected = [] 18 19 def matchTest(path): 20 match = re.search("(\.[\w\/\-_]*)\/.expect\/([\w\-_]+)(\.[\w\-_]+)?\.txt", path) 21 if match : 22 test = Test() 23 test.name = match.group(2) 24 test.path = match.group(1) 25 test.arch = match.group(3)[1:] if match.group(3) else None 26 if settings.arch.match(test.arch): 27 expected.append(test) 28 29 pathWalk( matchTest ) 30 31 return expected 21 # Test class that defines what a test is 22 class Test: 23 def __init__(self, name, path): 24 self.name, self.path = name, path 25 26 class TestResult: 27 SUCCESS = 0 28 FAILURE = 1 29 TIMEOUT = 124 30 31 # parses the Makefile to find the machine type (32-bit / 64-bit) 32 def getMachineType(): 33 sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c') 34 ret, out = sh("make .dummy -s", print2stdout=True) 35 36 if ret != 0: 37 print("Failed to identify architecture:") 38 print(out) 39 print("Stopping") 40 rm( (".dummy.c",".dummy") ) 41 sys.exit(1) 42 43 _, out = sh("file .dummy", print2stdout=False) 44 rm( (".dummy.c",".dummy") ) 45 46 return re.search("ELF\s([0-9]+)-bit", out).group(1) 47 48 def listTestsFolder(folder) : 49 path = ('./.expect/%s/' % folder) if folder else './.expect/' 50 subpath = "%s/" % folder if folder else "" 51 52 # tests directly in the .expect folder will always be processed 53 return map(lambda fname: Test(fname, subpath + fname), 54 [splitext(f)[0] for f in listdir( path ) 55 if not f.startswith('.') and f.endswith('.txt') 56 ]) 32 57 33 58 # reads the directory ./.expect and indentifies the tests 34 def listTests( includes, excludes ): 35 includes = [canonicalPath( i ) for i in includes] if includes else None 36 excludes = [canonicalPath( i ) for i in excludes] if excludes else None 59 def listTests( concurrent ): 60 machineType = getMachineType() 37 61 38 62 # tests directly in the .expect folder will always be processed 39 test_list = findTests() 40 41 # if we have a limited number of includes, filter by them 42 if includes: 43 test_list = [x for x in test_list if 44 x.path.startswith( tuple(includes) ) 45 ] 46 47 # # if we have a folders to excludes, filter by them 48 if excludes: 49 test_list = [x for x in test_list if not 50 x.path.startswith( tuple(excludes) ) 51 ] 52 53 return test_list 63 generic_list = listTestsFolder( "" ) 64 65 # tests in the machineType folder will be ran only for the corresponding compiler 66 typed_list = listTestsFolder( machineType ) 67 68 # tests in the concurrent folder will be ran only if concurrency is enabled 69 concurrent_list = listTestsFolder( "concurrent" ) if concurrent else [] 70 71 # append both lists to get 72 return generic_list + typed_list + concurrent_list; 54 73 55 74 # from the found tests, filter all the valid tests/desired tests … … 61 80 if options.regenerate_expected : 62 81 for testname in options.tests : 63 if Test.valid_name(testname): 64 found = [test for test in allTests if test.target() == testname] 65 tests.append( found[0] if len(found) == 1 else Test.from_target(testname) ) 82 if testname.endswith( (".c", ".cc", ".cpp") ): 83 print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr) 66 84 else : 67 print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr) 85 found = [test for test in allTests if test.name == testname] 86 tests.append( found[0] if len(found) == 1 else Test(testname, testname) ) 68 87 69 88 else : 70 89 # otherwise we only need to validate that all tests are present in the complete list 71 90 for testname in options.tests: 72 test = [t for t in allTests if pathCmp( t.target(), testname )]73 74 if test:91 test = [t for t in allTests if t.name == testname] 92 93 if len(test) != 0 : 75 94 tests.append( test[0] ) 76 95 else : … … 78 97 79 98 # make sure we have at least some test to run 80 if not tests:99 if len(tests) == 0 : 81 100 print('ERROR: No valid test to run', file=sys.stderr) 82 101 sys.exit(1) … … 89 108 parser = argparse.ArgumentParser(description='Script which runs cforall tests') 90 109 parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no') 91 parser.add_argument('-- arch', help='Test for specific architecture', type=str, default='')110 parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes') 92 111 parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true') 93 112 parser.add_argument('--list', help='List all test available', action='store_true') … … 96 115 parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8') 97 116 parser.add_argument('--list-comp', help='List all valide arguments', action='store_true') 98 parser.add_argument('-I','--include', help='Directory of test to include, can be used multiple time, All if omitted', action='append')99 parser.add_argument('-E','--exclude', help='Directory of test to exclude, can be used multiple time, None if omitted', action='append')100 117 parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run') 101 118 … … 106 123 all_tests = options.all 107 124 some_tests = len(options.tests) > 0 108 some_dirs = len(options.include) > 0 if options.include else 0109 125 110 126 # check that exactly one of the booleans is set to true 111 if not sum( (listing, all_tests, some_tests , some_dirs) ) > 0:112 print('ERROR: must have option \'--all\', \'--list\' , \'--include\', \'-I\'or non-empty test list', file=sys.stderr)127 if not sum( (listing, all_tests, some_tests) ) == 1 : 128 print('ERROR: must have option \'--all\', \'--list\' or non-empty test list', file=sys.stderr) 113 129 parser.print_help() 114 130 sys.exit(1) … … 116 132 return options 117 133 134 def jobCount( options ): 135 # check if the user already passed in a number of jobs for multi-threading 136 make_flags = environ.get('MAKEFLAGS') 137 make_jobs_fds = re.search("--jobserver-(auth|fds)=\s*([0-9]+),([0-9]+)", make_flags) if make_flags else None 138 if make_jobs_fds : 139 tokens = os.read(int(make_jobs_fds.group(2)), 1024) 140 options.jobs = len(tokens) 141 os.write(int(make_jobs_fds.group(3)), tokens) 142 else : 143 options.jobs = multiprocessing.cpu_count() 144 145 # make sure we have a valid number of jobs that corresponds to user input 146 if options.jobs <= 0 : 147 print('ERROR: Invalid number of jobs', file=sys.stderr) 148 sys.exit(1) 149 150 return min( options.jobs, len(tests) ), True if make_flags else False 151 118 152 ################################################################################ 119 153 # running test functions 120 154 ################################################################################ 121 155 # logic to run a single test and return the result (No handling of printing or other test framework logic) 122 def run_single_test(test ):156 def run_single_test(test, generate, dry_run, debug): 123 157 124 158 # find the output file based on the test name and options flag 125 out_file = test.target_output() 126 err_file = test.error_log() 127 cmp_file = test.expect() 128 in_file = test.input() 129 130 # prepare the proper directories 131 test.prepare() 159 out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path) 160 err_file = ".err/%s.log" % test.name 132 161 133 162 # remove any outputs from the previous tests to prevent side effects 134 rm( (out_file, err_file, test.target()) ) 163 rm( (out_file, err_file, test.name), dry_run ) 164 165 options = "-debug" if debug else "-nodebug" 135 166 136 167 # build, skipping to next test on error 137 make_ret, _ = make( test.target(),138 redirects = "2> %s 1> /dev/null" % out_file, 139 error_file = err_file140 )168 make_ret, _ = sh("""%s test=yes DEBUG_FLAGS="%s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run) 169 170 retcode = 0 171 error = None 141 172 142 173 # if the make command succeds continue otherwise skip to diff 143 if make_ret == 0 or settings.dry_run: 144 if settings.dry_run or fileIsExecutable(test.target()) : 174 if make_ret == 0 : 175 # fetch optional input 176 stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.name) else "" 177 178 if fileIsExecutable(test.name) : 145 179 # run test 146 retcode, _ = sh("timeout 60 %s > %s 2>&1" % (test.target(), out_file), input = in_file)180 retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run) 147 181 else : 148 182 # simply cat the result into the output 149 retcode, _ = sh("cat %s > %s" % (test.target(), out_file)) 150 else: 151 retcode, _ = sh("mv %s %s" % (err_file, out_file)) 152 183 sh("cat %s > %s" % (test.name, out_file), dry_run) 184 185 else : 186 # command failed save the log to less temporary file 187 sh("mv %s %s" % (err_file, out_file), dry_run) 153 188 154 189 if retcode == 0: 155 if settings.generating:190 if generate : 156 191 # if we are ounly generating the output we still need to check that the test actually exists 157 if not settings.dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'. Stop." % test.target()) :192 if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'. Stop." % test.name) : 158 193 retcode = 1; 159 error = "\t\tNo make target for test %s!" % test. target()194 error = "\t\tNo make target for test %s!" % test.name 160 195 sh("rm %s" % out_file, False) 161 else:162 error = None163 196 else : 164 197 # fetch return code and error from the diff command 165 retcode, error = diff( cmp_file, out_file)198 retcode, error = diff(".expect/%s.txt" % test.path, ".out/%s.log" % test.name, dry_run) 166 199 167 200 else: … … 171 204 172 205 # clean the executable 173 sh("rm -f %s > /dev/null 2>&1" % test. target())206 sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run) 174 207 175 208 return retcode, error 176 209 177 210 # run a single test and handle the errors, outputs, printing, exception handling, etc. 178 def run_test_worker(t) : 179 180 with SignalHandling(): 181 # print formated name 182 name_txt = "%20s " % t.name 183 184 retcode, error = run_single_test(t) 185 186 # update output based on current action 187 result_txt = TestResult.toString( retcode ) 188 189 #print result with error if needed 190 text = name_txt + result_txt 191 out = sys.stdout 192 if error : 193 text = text + "\n" + error 194 out = sys.stderr 195 196 print(text, file = out) 197 sys.stdout.flush() 198 sys.stderr.flush() 211 def run_test_worker(t, generate, dry_run, debug) : 212 213 signal.signal(signal.SIGINT, signal.SIG_DFL) 214 # print formated name 215 name_txt = "%20s " % t.name 216 217 retcode, error = run_single_test(t, generate, dry_run, debug) 218 219 # update output based on current action 220 if generate : 221 if retcode == TestResult.SUCCESS: result_txt = "Done" 222 elif retcode == TestResult.TIMEOUT: result_txt = "TIMEOUT" 223 else : result_txt = "ERROR code %d" % retcode 224 else : 225 if retcode == TestResult.SUCCESS: result_txt = "PASSED" 226 elif retcode == TestResult.TIMEOUT: result_txt = "TIMEOUT" 227 else : result_txt = "FAILED with code %d" % retcode 228 229 #print result with error if needed 230 text = name_txt + result_txt 231 out = sys.stdout 232 if error : 233 text = text + "\n" + error 234 out = sys.stderr 235 236 print(text, file = out) 237 sys.stdout.flush() 238 sys.stderr.flush() 239 signal.signal(signal.SIGINT, signal.SIG_IGN) 199 240 200 241 return retcode != TestResult.SUCCESS 201 242 202 243 # run the given list of tests with the given parameters 203 def run_tests(tests, jobs) :244 def run_tests(tests, generate, dry_run, jobs, debug) : 204 245 # clean the sandbox from previous commands 205 make('clean', redirects = '> /dev/null 2>&1') 246 sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run) 247 248 # make sure the required folder are present 249 sh('mkdir -p .out .expect .err', dry_run) 250 251 if generate : 252 print( "Regenerate tests for: " ) 206 253 207 254 # create the executor for our jobs and handle the signal properly 208 pool = setupPool(jobs) 255 original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) 256 pool = Pool(jobs) 257 signal.signal(signal.SIGINT, original_sigint_handler) 209 258 210 259 # for each test to run 211 260 try : 212 results = pool.map_async( 213 run_test_worker, 214 tests, 215 chunksize = 1 216 ).get(7200) 261 results = pool.map_async(partial(run_test_worker, generate=generate, dry_run=dry_run, debug=debug), tests, chunksize = 1 ).get(7200) 217 262 except KeyboardInterrupt: 218 263 pool.terminate() … … 221 266 222 267 # clean the workspace 223 make('clean', redirects = '> /dev/null 2>&1')268 sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run) 224 269 225 270 for failed in results: … … 240 285 options = getOptions() 241 286 242 # init global settings243 settings.init( options )244 245 287 # fetch the liest of all valid tests 246 allTests = listTests( options. include, options.exclude)288 allTests = listTests( options.concurrent ) 247 289 248 290 # if user wants all tests than no other treatement of the test list is required 249 if options.all or options.list or options.list_comp or options.include:291 if options.all or options.list or options.list_comp : 250 292 tests = allTests 251 293 252 #otherwise we need to validate that the test list that was entered is valid253 else :294 else : 295 #otherwise we need to validate that the test list that was entered is valid 254 296 tests = validTests( options ) 255 297 256 298 # sort the test alphabetically for convenience 257 tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())299 tests.sort(key=lambda t: t.name) 258 300 259 301 # users may want to simply list the tests 260 302 if options.list_comp : 261 print("-h --help --debug -- dry-run --list --arch--all --regenerate-expected -j --jobs ", end='')262 print(" ".join(map(lambda t: "%s" % (t. target()), tests)))303 print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='') 304 print(" ".join(map(lambda t: "%s" % (t.name), tests))) 263 305 264 306 elif options.list : 265 print("Listing for %s:%s"% (settings.arch.string, settings.debug.string)) 266 print("\n".join(map(lambda t: "%s" % (t.toString()), tests))) 267 268 else : 269 options.jobs, forceJobs = jobCount( options, tests ) 270 settings.updateMakeCmd(forceJobs, options.jobs) 271 272 print('%s (%s:%s) on %i cores' % ( 273 'Regenerate tests' if settings.generating else 'Running', 274 settings.arch.string, 275 settings.debug.string, 276 options.jobs 277 )) 307 print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests))) 308 309 else : 310 options.jobs, forceJobs = jobCount( options ) 311 312 print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs)) 313 make_cmd = "make" if forceJobs else ("make -j%i" % options.jobs) 278 314 279 315 # otherwise run all tests and make sure to return the correct error code 280 sys.exit( run_tests(tests, options. jobs) )316 sys.exit( run_tests(tests, options.regenerate_expected, options.dry_run, options.jobs, options.debug) )
Note:
See TracChangeset
for help on using the changeset viewer.
