source: tests/test.py@ 057298e

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 057298e was 0f3d844, checked in by Andrew Beach <ajbeach@…>, 5 years ago

Updates to cfa.nanorc. Added information on how to do some auto-completion in bash.

  • Property mode set to 100755
File size: 12.5 KB
Line 
1#!/usr/bin/python3
2
3from pybin.tools import *
4from pybin.test_run import *
5from pybin import settings
6
7import argparse
8import itertools
9import re
10import sys
11import tempfile
12import time
13
14import os
15import psutil
16import signal
17
18################################################################################
19# help functions
20################################################################################
21
22def find_tests():
23 expected = []
24
25 def match_test(path):
26 match = re.search("^%s\/([\w\/\-_]*).expect\/([\w\-_]+)(\.[\w\-_]+)?\.txt$" % settings.SRCDIR, path)
27 if match :
28 test = Test()
29 test.name = match.group(2)
30 test.path = match.group(1)
31 test.arch = match.group(3)[1:] if match.group(3) else None
32 expected.append(test)
33
34 path_walk( match_test )
35
36 return expected
37
38# reads the directory ./.expect and indentifies the tests
39def list_tests( includes, excludes ):
40 # tests directly in the .expect folder will always be processed
41 test_list = find_tests()
42
43 # if we have a limited number of includes, filter by them
44 if includes:
45 test_list = [x for x in test_list if
46 x.target().startswith( tuple(includes) )
47 ]
48
49 # # if we have a folders to excludes, filter by them
50 if excludes:
51 test_list = [x for x in test_list if not
52 x.target().startswith( tuple(excludes) )
53 ]
54
55 # sort the test alphabetically for convenience
56 test_list.sort(key=lambda t: ('~' if t.arch else '') + t.target() + (t.arch if t.arch else ''))
57
58 return test_list
59
60# from the found tests, filter all the valid tests/desired tests
61def valid_tests( options ):
62 tests = []
63
64 # if we are regenerating the tests we need to find the information of the
65 # already existing tests and create new info for the new tests
66 if options.regenerate_expected :
67 for testname in options.tests :
68 testname = canonical_path( testname )
69 if Test.valid_name(testname):
70 found = [test for test in all_tests if canonical_path( test.target() ) == testname]
71 tests.append( found[0] if len(found) == 1 else Test.from_target(testname) )
72 else :
73 print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr)
74
75 else :
76 # otherwise we only need to validate that all tests are present in the complete list
77 for testname in options.tests:
78 test = [t for t in all_tests if path_cmp( t.target(), testname )]
79
80 if test :
81 tests.extend( test )
82 else :
83 print('ERROR: No expected file for test %s, ignoring it' % testname, file=sys.stderr)
84
85 return tests
86
87# parses the option
88def parse_args():
89 # create a parser with the arguments for the tests script
90 parser = argparse.ArgumentParser(description='Script which runs cforall tests')
91 parser.add_argument('--debug', help='Run all tests in debug or release', type=comma_separated(yes_no), default='yes')
92 parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=comma_separated(yes_no), default='no')
93 parser.add_argument('--arch', help='Test for specific architecture', type=comma_separated(str), default='')
94 parser.add_argument('--continue', help='When multiple specifications are passed (debug/install/arch), sets whether or not to continue if the last specification failed', type=yes_no, default='yes', dest='continue_')
95 parser.add_argument('--timeout', help='Maximum duration in seconds after a single test is considered to have timed out', type=int, default=60)
96 parser.add_argument('--global-timeout', help='Maximum cumulative duration in seconds after the ALL tests are considered to have timed out', type=int, default=7200)
97 parser.add_argument('--timeout-with-gdb', help='Instead of killing the command when it times out, orphan it and print process id to allow gdb to attach', type=yes_no, default="no")
98 parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
99 parser.add_argument('--list', help='List all test available', action='store_true')
100 parser.add_argument('--all', help='Run all test available', action='store_true')
101 parser.add_argument('--regenerate-expected', help='Regenerate the .expect by running the specified tets, can be used with --all option', action='store_true')
102 parser.add_argument('--archive-errors', help='If called with a valid path, on test crashes the test script will copy the core dump and the executable to the specified path.', type=str, default='')
103 parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int)
104 parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
105 parser.add_argument('-I','--include', help='Directory of test to include, can be used multiple time, All if omitted', action='append')
106 parser.add_argument('-E','--exclude', help='Directory of test to exclude, can be used multiple time, None if omitted', action='append')
107 parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
108
109 try:
110 options = parser.parse_args()
111 except:
112 print('ERROR: invalid arguments', file=sys.stderr)
113 parser.print_help(sys.stderr)
114 sys.exit(1)
115
116 # script must have at least some tests to run or be listing
117 listing = options.list or options.list_comp
118 all_tests = options.all
119 some_tests = len(options.tests) > 0
120 some_dirs = len(options.include) > 0 if options.include else 0
121
122 # check that exactly one of the booleans is set to true
123 if not sum( (listing, all_tests, some_tests, some_dirs) ) > 0 :
124 print('''ERROR: must have option '--all', '--list', '--include', '-I' or non-empty test list''', file=sys.stderr)
125 parser.print_help()
126 sys.exit(1)
127
128 return options
129
130################################################################################
131# running test functions
132################################################################################
133def success(val):
134 return val == 0 or settings.dry_run
135
136def no_rule(file, target):
137 return not settings.dry_run and file_contains_only(file, "make: *** No rule to make target `%s'. Stop." % target)
138
139# logic to run a single test and return the result (No handling of printing or other test framework logic)
140def run_single_test(test):
141
142 # find the output file based on the test name and options flag
143 exe_file = test.target_executable();
144 out_file = test.target_output()
145 err_file = test.error_log()
146 cmp_file = test.expect()
147 in_file = test.input()
148
149 # prepare the proper directories
150 test.prepare()
151
152 # build, skipping to next test on error
153 with Timed() as comp_dur:
154 make_ret, _ = make( test.target(), output_file=subprocess.DEVNULL, error=out_file, error_file = err_file )
155
156 run_dur = None
157 # run everything in a temp directory to make sure core file are handled properly
158 with tempdir():
159 # if the make command succeeds continue otherwise skip to diff
160 if success(make_ret):
161 with Timed() as run_dur:
162 if settings.dry_run or is_exe(exe_file):
163 # run test
164 retcode, _ = sh(exe_file, output_file=out_file, input_file=in_file, timeout=True)
165 else :
166 # simply cat the result into the output
167 retcode = cat(exe_file, out_file)
168 else:
169 retcode = mv(err_file, out_file)
170
171 if success(retcode):
172 if settings.generating :
173 # if we are ounly generating the output we still need to check that the test actually exists
174 if no_rule(out_file, test.target()) :
175 retcode = 1
176 error = "\t\tNo make target for test %s!" % test.target()
177 rm(out_file)
178 else:
179 error = None
180 else :
181 # fetch return code and error from the diff command
182 retcode, error = diff(cmp_file, out_file)
183
184 else:
185 with open (out_file, "r") as myfile:
186 error = myfile.read()
187
188 ret, info = core_info(exe_file)
189 error = error + info if error else info
190
191 if settings.archive:
192 error = error + '\n' + core_archive(settings.archive, test.target(), exe_file)
193
194
195
196 # clean the executable
197 rm(exe_file)
198
199 return retcode, error, [comp_dur.duration, run_dur.duration if run_dur else None]
200
201# run a single test and handle the errors, outputs, printing, exception handling, etc.
202def run_test_worker(t) :
203 try :
204 # print formated name
205 name_txt = '{0:{width}} '.format(t.target(), width=settings.output_width)
206
207 retcode, error, duration = run_single_test(t)
208
209 # update output based on current action
210 result_txt = TestResult.toString( retcode, duration )
211
212 #print result with error if needed
213 text = '\t' + name_txt + result_txt
214 out = sys.stdout
215 if error :
216 text = text + '\n' + error
217
218 return retcode == TestResult.SUCCESS, text
219 except KeyboardInterrupt:
220 return False, ""
221 except Exception as ex:
222 print("Unexpected error in worker thread: %s" % ex, file=sys.stderr)
223 sys.stderr.flush()
224 return False, ""
225
226
227# run the given list of tests with the given parameters
228def run_tests(tests, jobs) :
229 # clean the sandbox from previous commands
230 make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
231
232 # since python prints stacks by default on a interrupt, redo the interrupt handling to be silent
233 def worker_init():
234 def sig_int(signal_num, frame):
235 pass
236
237 signal.signal(signal.SIGINT, sig_int)
238
239 # create the executor for our jobs and handle the signal properly
240 pool = multiprocessing.Pool(jobs, worker_init)
241
242 failed = False
243
244 def stop(x, y):
245 print("Tests interrupted by user", file=sys.stderr)
246 sys.exit(1)
247 signal.signal(signal.SIGINT, stop)
248
249 # for each test to run
250 try :
251 num = len(tests)
252 fancy = sys.stdout.isatty()
253 results = pool.imap_unordered(
254 run_test_worker,
255 tests,
256 chunksize = 1
257 )
258
259 for i, (succ, txt) in enumerate(timed(results, timeout = settings.timeout.total), 1) :
260 if not succ :
261 failed = True
262
263 print(" " + txt)
264
265 if(fancy and i != num):
266 print("%d/%d" % (i, num), end='\r')
267 sys.stdout.flush()
268
269 except KeyboardInterrupt:
270 print("Tests interrupted by user", file=sys.stderr)
271 pool.terminate()
272 pool.join()
273 failed = True
274 except multiprocessing.TimeoutError:
275 print("ERROR: Test suite timed out", file=sys.stderr)
276 pool.terminate()
277 pool.join()
278 failed = True
279 killgroup() # needed to cleanly kill all children
280
281
282 # clean the workspace
283 make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
284
285 return failed
286
287
288################################################################################
289# main loop
290################################################################################
291if __name__ == "__main__":
292
293 # parse the command line arguments
294 options = parse_args()
295
296 # init global settings
297 settings.init( options )
298
299 # users may want to simply list the tests
300 if options.list_comp :
301 # fetch the liest of all valid tests
302 tests = list_tests( None, None )
303
304 # print the possible options
305 print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs -I --include -E --exclude --continue ", end='')
306 print(" ".join(map(lambda t: "%s" % (t.target()), tests)))
307
308 elif options.list :
309 # fetch the liest of all valid tests
310 tests = list_tests( options.include, options.exclude )
311
312 # print the available tests
313 fancy_print("\n".join(map(lambda t: t.toString(), tests)))
314
315 else :
316 # fetch the liest of all valid tests
317 all_tests = list_tests( options.include, options.exclude )
318
319 # if user wants all tests than no other treatement of the test list is required
320 if options.all or options.include :
321 tests = all_tests
322
323 #otherwise we need to validate that the test list that was entered is valid
324 else :
325 tests = valid_tests( options )
326
327 # make sure we have at least some test to run
328 if not tests :
329 print('ERROR: No valid test to run', file=sys.stderr)
330 sys.exit(1)
331
332 # prep invariants
333 settings.prep_output(tests)
334 failed = 0
335
336 # for each build configurations, run the test
337 for arch, debug, install in itertools.product(settings.all_arch, settings.all_debug, settings.all_install):
338 settings.arch = arch
339 settings.debug = debug
340 settings.install = install
341
342 # filter out the tests for a different architecture
343 # tests are the same across debug/install
344 local_tests = settings.arch.filter( tests )
345 options.jobs, forceJobs = job_count( options, local_tests )
346 settings.update_make_cmd(forceJobs, options.jobs)
347
348 # check the build configuration works
349 settings.validate()
350
351 # print configuration
352 print('%s %i tests on %i cores (%s:%s)' % (
353 'Regenerating' if settings.generating else 'Running',
354 len(local_tests),
355 options.jobs,
356 settings.arch.string,
357 settings.debug.string
358 ))
359
360 # otherwise run all tests and make sure to return the correct error code
361 failed = run_tests(local_tests, options.jobs)
362 if failed:
363 result = 1
364 if not settings.continue_:
365 break
366
367
368 sys.exit( failed )
Note: See TracBrowser for help on using the repository browser.