- Timestamp:
- Dec 4, 2017, 1:19:42 PM (7 years ago)
- Branches:
- ADT, aaron-thesis, arm-eh, ast-experimental, cleanup-dtors, deferred_resn, demangler, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, new-env, no_list, persistent-indexer, pthread-emulation, qualifiedEnum, resolv-new, with_gc
- Children:
- bacc36c
- Parents:
- b0e5593
- Location:
- src/tests
- Files:
-
- 1 added
- 4 edited
- 1 moved
-
Makefile.am (modified) (4 diffs)
-
Makefile.in (modified) (4 diffs)
-
coroutine/.in/fmtLines.txt (moved) (moved from src/tests/.in/fmtLines.txt)
-
pybin/test_run.py (added)
-
pybin/tools.py (modified) (2 diffs)
-
test.py (modified) (13 diffs)
Legend:
- Unmodified
- Added
- Removed
-
src/tests/Makefile.am
rb0e5593 r0ad0c55 20 20 21 21 if BUILD_CONCURRENCY 22 concurrent = yes 23 quick_test += coroutine thread monitor 24 concurrent_test = \ 25 coroutine \ 26 fmtLines \ 27 pingpong \ 28 prodcons \ 29 thread \ 30 matrixSum \ 31 monitor \ 32 multi-monitor \ 33 boundedBuffer \ 34 preempt \ 35 sched-int-block \ 36 sched-int-disjoint \ 37 sched-int-wait \ 38 sched-ext-barge \ 39 sched-ext-dtor \ 40 sched-ext-else \ 41 sched-ext-parse \ 42 sched-ext-recurse \ 43 sched-ext-statment \ 44 sched-ext-when 22 concurrent= 45 23 else 46 concurrent=no 47 concurrent_test= 24 concurrent='-Econcurrent' 48 25 endif 49 26 … … 62 39 endif 63 40 64 TEST_FLAGS = $(if $(test), 2> .err/${@}.log, )41 TEST_FLAGS = $(if $(test), 2> $(test), ) 65 42 AM_CFLAGS = ${TEST_FLAGS} ${BUILD_FLAGS} 66 43 CC = @CFA_BINDIR@/@CFA_NAME@ … … 70 47 71 48 fstream_test_SOURCES = fstream_test.c 72 fstream_test_CFLAGS = $(if $(test), 2>> .err/fstream_test.log, ) ${BUILD_FLAGS}49 fstream_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS} 73 50 74 51 vector_test_SOURCES = vector/vector_int.c vector/array.c vector/vector_test.c 75 vector_test_CFLAGS = $(if $(test), 2>> .err/vector_test.log, ) ${BUILD_FLAGS}52 vector_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS} 76 53 77 54 avl_test_SOURCES = avltree/avl_test.c avltree/avl0.c avltree/avl1.c avltree/avl2.c avltree/avl3.c avltree/avl4.c avltree/avl-private.c 78 avl_test_CFLAGS = $(if $(test), 2>> .err/avl_test.log, ) ${BUILD_FLAGS}55 avl_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS} 79 56 80 57 all-local : 81 @+python test.py --debug=${debug} --concurrent=${concurrent} ${quick_test}58 @+python test.py --debug=${debug} ${concurrent} ${quick_test} 82 59 83 60 all-tests : 84 @+python test.py --all --debug=${debug} --concurrent=${concurrent} # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program61 @+python test.py --all --debug=${debug} ${concurrent} # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program 85 62 86 63 clean-local : … … 88 65 89 66 list : 90 @+python test.py --list --concurrent=${concurrent}67 @+python test.py --list ${concurrent} 91 68 92 69 concurrency : 93 @+python test.py --debug=${debug} --concurrent=${concurrent} ${concurrent_test}70 @+python test.py --debug=${debug} ${concurrent} ${concurrent_test} 94 71 95 72 .dummy : .dummy.c @CFA_BINDIR@/@CFA_NAME@ -
src/tests/Makefile.in
rb0e5593 r0ad0c55 91 91 build_triplet = @build@ 92 92 host_triplet = @host@ 93 @BUILD_CONCURRENCY_TRUE@am__append_1 = coroutine thread monitor 94 @BUILD_DEBUG_FALSE@am__append_2 = -nodebug 95 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_FALSE@am__append_3 = -debug 96 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_TRUE@am__append_4 = ${DEBUG_FLAGS} 93 @BUILD_DEBUG_FALSE@am__append_1 = -nodebug 94 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_FALSE@am__append_2 = -debug 95 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_TRUE@am__append_3 = ${DEBUG_FLAGS} 97 96 EXTRA_PROGRAMS = fstream_test$(EXEEXT) vector_test$(EXEEXT) \ 98 97 avl_test$(EXEEXT) … … 314 313 top_srcdir = @top_srcdir@ 315 314 debug = yes 316 quick_test = vector_test avl_test operators numericConstants \ 317 expression enum array typeof cast dtor-early-exit init_once \ 318 attributes $(am__append_1) 319 @BUILD_CONCURRENCY_FALSE@concurrent = no 320 @BUILD_CONCURRENCY_TRUE@concurrent = yes 321 @BUILD_CONCURRENCY_FALSE@concurrent_test = 322 @BUILD_CONCURRENCY_TRUE@concurrent_test = \ 323 @BUILD_CONCURRENCY_TRUE@ coroutine \ 324 @BUILD_CONCURRENCY_TRUE@ fmtLines \ 325 @BUILD_CONCURRENCY_TRUE@ pingpong \ 326 @BUILD_CONCURRENCY_TRUE@ prodcons \ 327 @BUILD_CONCURRENCY_TRUE@ thread \ 328 @BUILD_CONCURRENCY_TRUE@ matrixSum \ 329 @BUILD_CONCURRENCY_TRUE@ monitor \ 330 @BUILD_CONCURRENCY_TRUE@ multi-monitor \ 331 @BUILD_CONCURRENCY_TRUE@ boundedBuffer \ 332 @BUILD_CONCURRENCY_TRUE@ preempt \ 333 @BUILD_CONCURRENCY_TRUE@ sched-int-block \ 334 @BUILD_CONCURRENCY_TRUE@ sched-int-disjoint \ 335 @BUILD_CONCURRENCY_TRUE@ sched-int-wait \ 336 @BUILD_CONCURRENCY_TRUE@ sched-ext-barge \ 337 @BUILD_CONCURRENCY_TRUE@ sched-ext-dtor \ 338 @BUILD_CONCURRENCY_TRUE@ sched-ext-else \ 339 @BUILD_CONCURRENCY_TRUE@ sched-ext-parse \ 340 @BUILD_CONCURRENCY_TRUE@ sched-ext-recurse \ 341 @BUILD_CONCURRENCY_TRUE@ sched-ext-statment \ 342 @BUILD_CONCURRENCY_TRUE@ sched-ext-when 343 315 quick_test = vector_test avl_test operators numericConstants expression enum array typeof cast dtor-early-exit init_once attributes 316 @BUILD_CONCURRENCY_FALSE@concurrent = '-Econcurrent' 317 @BUILD_CONCURRENCY_TRUE@concurrent = 344 318 345 319 # applies to both programs 346 320 DEBUG_FLAGS = 347 321 BUILD_FLAGS = -g -Wall -Wno-unused-function -quiet @CFA_FLAGS@ \ 348 $(am__append_ 2) $(am__append_3) $(am__append_4)349 TEST_FLAGS = $(if $(test), 2> .err/${@}.log, )322 $(am__append_1) $(am__append_2) $(am__append_3) 323 TEST_FLAGS = $(if $(test), 2> $(test), ) 350 324 AM_CFLAGS = ${TEST_FLAGS} ${BUILD_FLAGS} 351 325 fstream_test_SOURCES = fstream_test.c 352 fstream_test_CFLAGS = $(if $(test), 2>> .err/fstream_test.log, ) ${BUILD_FLAGS}326 fstream_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS} 353 327 vector_test_SOURCES = vector/vector_int.c vector/array.c vector/vector_test.c 354 vector_test_CFLAGS = $(if $(test), 2>> .err/vector_test.log, ) ${BUILD_FLAGS}328 vector_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS} 355 329 avl_test_SOURCES = avltree/avl_test.c avltree/avl0.c avltree/avl1.c avltree/avl2.c avltree/avl3.c avltree/avl4.c avltree/avl-private.c 356 avl_test_CFLAGS = $(if $(test), 2>> .err/avl_test.log, ) ${BUILD_FLAGS}330 avl_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS} 357 331 all: all-am 358 332 … … 833 807 834 808 all-local : 835 @+python test.py --debug=${debug} --concurrent=${concurrent} ${quick_test}809 @+python test.py --debug=${debug} ${concurrent} ${quick_test} 836 810 837 811 all-tests : 838 @+python test.py --all --debug=${debug} --concurrent=${concurrent} # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program812 @+python test.py --all --debug=${debug} ${concurrent} # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program 839 813 840 814 clean-local : … … 842 816 843 817 list : 844 @+python test.py --list --concurrent=${concurrent}818 @+python test.py --list ${concurrent} 845 819 846 820 concurrency : 847 @+python test.py --debug=${debug} --concurrent=${concurrent} ${concurrent_test}821 @+python test.py --debug=${debug} ${concurrent} ${concurrent_test} 848 822 849 823 .dummy : .dummy.c @CFA_BINDIR@/@CFA_NAME@ -
src/tests/pybin/tools.py
rb0e5593 r0ad0c55 46 46 os.rename(out_fname, fname) 47 47 48 # helper function to check if a files contains only a sp acific string48 # helper function to check if a files contains only a specific string 49 49 def fileContainsOnly(file, text) : 50 50 with open(file) as f: … … 95 95 # fetch return code and error from the diff command 96 96 return sh(diff_cmd % (lhs, rhs), dry_run, False) 97 98 # parses the Makefile to find the machine type (32-bit / 64-bit) 99 def getMachineType(): 100 return 'x64' 101 sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c') 102 ret, out = sh("make .dummy -s", print2stdout=True) 103 104 if ret != 0: 105 print("Failed to identify architecture:") 106 print(out) 107 print("Stopping") 108 rm( (".dummy.c",".dummy") ) 109 sys.exit(1) 110 111 _, out = sh("file .dummy", print2stdout=False) 112 rm( (".dummy.c",".dummy") ) 113 114 return out 115 return re.search("ELF\s([0-9]+)-bit", out).group(1) -
src/tests/test.py
rb0e5593 r0ad0c55 7 7 from os.path import isfile, join, splitext 8 8 from pybin.tools import * 9 from pybin.test_run import * 9 10 10 11 import argparse … … 19 20 ################################################################################ 20 21 21 # Test class that defines what a test is 22 class Test: 23 def __init__(self, name, path): 24 self.name, self.path = name, path 25 26 class TestResult: 27 SUCCESS = 0 28 FAILURE = 1 29 TIMEOUT = 124 30 31 # parses the Makefile to find the machine type (32-bit / 64-bit) 32 def getMachineType(): 33 sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c') 34 ret, out = sh("make .dummy -s", print2stdout=True) 35 36 if ret != 0: 37 print("Failed to identify architecture:") 38 print(out) 39 print("Stopping") 40 rm( (".dummy.c",".dummy") ) 41 sys.exit(1) 42 43 _, out = sh("file .dummy", print2stdout=False) 44 rm( (".dummy.c",".dummy") ) 45 46 return re.search("ELF\s([0-9]+)-bit", out).group(1) 47 48 def listTestsFolder(folder) : 49 path = ('./.expect/%s/' % folder) if folder else './.expect/' 50 subpath = "%s/" % folder if folder else "" 22 def list_expected(): 23 expected = [] 24 25 def step(_, dirname, names): 26 for name in names: 27 path = os.path.join(dirname, name) 28 29 match = re.search("(\.[\w\/\-_]*)\/.expect\/([\w\-_]+)(\.[\w\-_]+)?\.txt", path) 30 if match : 31 test = Test() 32 test.name = match.group(2) 33 test.path = match.group(1) 34 test.arch = match.group(3)[1:] if match.group(3) else None 35 expected.append(test) 36 37 # Start the walk 38 os.path.walk('.', step, '') 39 40 return expected 41 42 # reads the directory ./.expect and indentifies the tests 43 def listTests( includes, excludes ): 44 includes = [os.path.normpath( os.path.join('.',i) ) for i in includes] if includes else None 45 excludes = [os.path.normpath( os.path.join('.',i) ) for i in excludes] if excludes else None 51 46 52 47 # tests directly in the .expect folder will always be processed 53 return map(lambda fname: Test(fname, subpath + fname), 54 [splitext(f)[0] for f in listdir( path ) 55 if not f.startswith('.') and f.endswith('.txt') 56 ]) 57 58 # reads the directory ./.expect and indentifies the tests 59 def listTests( concurrent ): 60 machineType = getMachineType() 61 62 # tests directly in the .expect folder will always be processed 63 generic_list = listTestsFolder( "" ) 64 65 # tests in the machineType folder will be ran only for the corresponding compiler 66 typed_list = listTestsFolder( machineType ) 67 68 # tests in the concurrent folder will be ran only if concurrency is enabled 69 concurrent_list = listTestsFolder( "concurrent" ) if concurrent else [] 70 71 # append both lists to get 72 return generic_list + typed_list + concurrent_list; 48 test_list = list_expected() 49 50 # if we have a limited number of includes, filter by them 51 if includes: 52 test_list = [x for x in test_list if 53 os.path.normpath( x.path ).startswith( tuple(includes) ) 54 ] 55 56 # # if we have a folders to excludes, filter by them 57 if excludes: 58 test_list = [x for x in test_list if not 59 os.path.normpath( x.path ).startswith( tuple(excludes) ) 60 ] 61 62 return test_list 73 63 74 64 # from the found tests, filter all the valid tests/desired tests … … 89 79 # otherwise we only need to validate that all tests are present in the complete list 90 80 for testname in options.tests: 91 test = [t for t in allTests if t.name == testname]81 test = [t for t in allTests if os.path.normpath( t.target() ) == os.path.normpath( testname )] 92 82 93 83 if len(test) != 0 : … … 102 92 103 93 return tests 94 95 class TestResult: 96 SUCCESS = 0 97 FAILURE = 1 98 TIMEOUT = 124 104 99 105 100 # parses the option … … 108 103 parser = argparse.ArgumentParser(description='Script which runs cforall tests') 109 104 parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no') 110 parser.add_argument('-- concurrent', help='Run concurrent tests', type=yes_no, default='yes')105 parser.add_argument('--arch', help='Test for specific architecture', type=str, default=getMachineType()) 111 106 parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true') 112 107 parser.add_argument('--list', help='List all test available', action='store_true') … … 115 110 parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8') 116 111 parser.add_argument('--list-comp', help='List all valide arguments', action='store_true') 112 parser.add_argument('-I','--include', help='Directory of test to include, can be used multiple time, All if omitted', action='append') 113 parser.add_argument('-E','--exclude', help='Directory of test to exclude, can be used multiple time, None if omitted', action='append') 117 114 parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run') 118 115 … … 123 120 all_tests = options.all 124 121 some_tests = len(options.tests) > 0 122 some_dirs = len(options.include) > 0 if options.include else 0 125 123 126 124 # check that exactly one of the booleans is set to true 127 if not sum( (listing, all_tests, some_tests ) ) == 1:128 print('ERROR: must have option \'--all\', \'--list\' or non-empty test list', file=sys.stderr)125 if not sum( (listing, all_tests, some_tests, some_dirs) ) > 0 : 126 print('ERROR: must have option \'--all\', \'--list\', \'--include\', \'-I\' or non-empty test list', file=sys.stderr) 129 127 parser.print_help() 130 128 sys.exit(1) … … 157 155 158 156 # find the output file based on the test name and options flag 159 out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path) 160 err_file = ".err/%s.log" % test.name 157 out_file = test.output_file() if not generate else test.expect_file() 158 err_file = test.error_file() 159 cmp_file = test.expect_file() 160 in_file = test.input_file() 161 162 # prepare the proper directories 163 test.prepare( dry_run ) 161 164 162 165 # remove any outputs from the previous tests to prevent side effects 163 rm( (out_file, err_file, test. name), dry_run )166 rm( (out_file, err_file, test.target()), dry_run ) 164 167 165 168 options = "-debug" if debug else "-nodebug" 166 169 170 167 171 # build, skipping to next test on error 168 make_ret, _ = sh("""%s test=yes DEBUG_FLAGS="%s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)172 make_ret, _ = sh("""%s DEBUG_FLAGS="%s" %s test="%s" 2> %s 1> /dev/null""" % (make_cmd, options, test.target(), err_file, out_file), dry_run) 169 173 170 174 retcode = 0 … … 172 176 173 177 # if the make command succeds continue otherwise skip to diff 174 if make_ret == 0 :178 if make_ret == 0 or dry_run: 175 179 # fetch optional input 176 stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.name) else ""177 178 if fileIsExecutable(test.name) :180 stdinput = "< %s" % in_file if isfile(in_file) else "" 181 182 if dry_run or fileIsExecutable(test.target()) : 179 183 # run test 180 retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test. name, stdinput, out_file), dry_run)184 retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test.target(), stdinput, out_file), dry_run) 181 185 else : 182 186 # simply cat the result into the output 183 sh("cat %s > %s" % (test.name, out_file), dry_run) 184 185 else : 186 # command failed save the log to less temporary file 187 sh("cat %s > %s" % (test.target(), out_file), dry_run) 188 else: 187 189 sh("mv %s %s" % (err_file, out_file), dry_run) 190 188 191 189 192 if retcode == 0: 190 193 if generate : 191 194 # if we are ounly generating the output we still need to check that the test actually exists 192 if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'. Stop." % test. name) :195 if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'. Stop." % test.target()) : 193 196 retcode = 1; 194 error = "\t\tNo make target for test %s!" % test. name197 error = "\t\tNo make target for test %s!" % test.target() 195 198 sh("rm %s" % out_file, False) 196 199 else : 197 200 # fetch return code and error from the diff command 198 retcode, error = diff( ".expect/%s.txt" % test.path, ".out/%s.log" % test.name, dry_run)201 retcode, error = diff(cmp_file, out_file, dry_run) 199 202 200 203 else: … … 204 207 205 208 # clean the executable 206 sh("rm -f %s > /dev/null 2>&1" % test. name, dry_run)209 sh("rm -f %s > /dev/null 2>&1" % test.target(), dry_run) 207 210 208 211 return retcode, error … … 245 248 # clean the sandbox from previous commands 246 249 sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run) 247 248 # make sure the required folder are present249 sh('mkdir -p .out .expect .err', dry_run)250 250 251 251 if generate : … … 286 286 287 287 # fetch the liest of all valid tests 288 allTests = listTests( options. concurrent)288 allTests = listTests( options.include, options.exclude ) 289 289 290 290 # if user wants all tests than no other treatement of the test list is required 291 if options.all or options.list or options.list_comp :291 if options.all or options.list or options.list_comp or options.include : 292 292 tests = allTests 293 293 … … 297 297 298 298 # sort the test alphabetically for convenience 299 tests.sort(key=lambda t: t.name)299 tests.sort(key=lambda t: os.path.join(t.path, t.name)) 300 300 301 301 # users may want to simply list the tests 302 302 if options.list_comp : 303 print("-h --help --debug -- concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')304 print(" ".join(map(lambda t: "%s" % (t. name), tests)))303 print("-h --help --debug --dry-run --list --all --regenerate-expected -j --jobs ", end='') 304 print(" ".join(map(lambda t: "%s" % (t.target()), tests))) 305 305 306 306 elif options.list : 307 print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests))) 307 print("Listing for %s:%s"% (options.arch, "debug" if options.debug else "no debug")) 308 print("\n".join(map(lambda t: "%s" % (t.toString()), tests))) 308 309 309 310 else : 310 311 options.jobs, forceJobs = jobCount( options ) 311 312 312 print('Running (%s ) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))313 print('Running (%s:%s) on %i cores' % (options.arch, "debug" if options.debug else "no debug", options.jobs)) 313 314 make_cmd = "make" if forceJobs else ("make -j%i" % options.jobs) 314 315
Note: See TracChangeset
for help on using the changeset viewer.
