Changeset 3d5701e for tests/pybin/tools.py
- Timestamp:
- Feb 25, 2020, 1:17:33 PM (6 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- 7dc2e015
- Parents:
- 9fb8f01 (diff), dd9e1ca (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)links above to see all the changes relative to each parent. - File:
-
- 1 edited
-
tests/pybin/tools.py (modified) (11 diffs)
Legend:
- Unmodified
- Added
- Removed
-
tests/pybin/tools.py
r9fb8f01 r3d5701e 23 23 24 24 # helper functions to run terminal commands 25 def sh(*cmd, timeout = False, output = None, input = None, error = subprocess.STDOUT): 26 cmd = list(cmd) 27 28 # if this is a dry_run, only print the commands that would be ran 29 if settings.dry_run : 30 cmd = "{} cmd: {}".format(os.getcwd(), ' '.join(cmd)) 31 if output and not isinstance(output, int): 32 cmd += " > " 33 cmd += output 34 35 if error and not isinstance(error, int): 36 cmd += " 2> " 37 cmd += error 38 39 if input and not isinstance(input, int) and os.path.isfile(input): 40 cmd += " < " 41 cmd += input 42 43 print(cmd) 44 return 0, None 45 46 with contextlib.ExitStack() as onexit: 47 # add input redirection if needed 48 input = openfd(input, 'r', onexit, True) 49 50 # add output redirection if needed 51 output = openfd(output, 'w', onexit, False) 52 53 # add error redirection if needed 54 error = openfd(error, 'w', onexit, False) 55 56 # run the desired command 57 try: 58 proc = subprocess.run( 25 def sh(*cmd, timeout = False, output_file = None, input_file = None, input_text = None, error = subprocess.STDOUT, ignore_dry_run = False): 26 try: 27 cmd = list(cmd) 28 29 if input_file and input_text: 30 return 401, "Cannot use both text and file inputs" 31 32 # if this is a dry_run, only print the commands that would be ran 33 if settings.dry_run and not ignore_dry_run: 34 cmd = "{} cmd: {}".format(os.getcwd(), ' '.join(cmd)) 35 if output_file and not isinstance(output_file, int): 36 cmd += " > " 37 cmd += output_file 38 39 if error and not isinstance(error, int): 40 cmd += " 2> " 41 cmd += error 42 43 if input_file and not isinstance(input_file, int) and os.path.isfile(input_file): 44 cmd += " < " 45 cmd += input_file 46 47 print(cmd) 48 return 0, None 49 50 with contextlib.ExitStack() as onexit: 51 # add input redirection if needed 52 input_file = openfd(input_file, 'r', onexit, True) 53 54 # add output redirection if needed 55 output_file = openfd(output_file, 'w', onexit, False) 56 57 # add error redirection if needed 58 error = openfd(error, 'w', onexit, False) 59 60 # run the desired command 61 # use with statement to make sure proc is cleaned 62 # don't use subprocess.run because we want to send SIGABRT on exit 63 with subprocess.Popen( 59 64 cmd, 60 stdin =input, 61 stdout=output, 62 stderr=error, 63 timeout=settings.timeout.single if timeout else None 64 ) 65 return proc.returncode, proc.stdout.decode("utf-8") if proc.stdout else None 66 except subprocess.TimeoutExpired: 67 return 124, str(None) 65 **({'input' : bytes(input_text, encoding='utf-8')} if input_text else {'stdin' : input_file}), 66 stdout = output_file, 67 stderr = error 68 ) as proc: 69 70 try: 71 out, _ = proc.communicate( 72 timeout = settings.timeout.single if timeout else None 73 ) 74 75 return proc.returncode, out.decode("utf-8") if out else None 76 except subprocess.TimeoutExpired: 77 proc.send_signal(signal.SIGABRT) 78 proc.communicate() 79 return 124, str(None) 80 81 except Exception as ex: 82 print ("Unexpected error: %s" % ex) 83 raise 68 84 69 85 def is_ascii(fname): … … 75 91 return False 76 92 77 code, out = sh("file %s" % fname, output =subprocess.PIPE)93 code, out = sh("file %s" % fname, output_file=subprocess.PIPE) 78 94 if code != 0: 79 95 return False … … 107 123 if isinstance(files, str ): files = [ files ] 108 124 for file in files: 109 sh( 'rm', '-f', file, output =subprocess.DEVNULL, error=subprocess.DEVNULL )125 sh( 'rm', '-f', file, output_file=subprocess.DEVNULL, error=subprocess.DEVNULL ) 110 126 111 127 # Create 1 or more directory … … 115 131 p = os.path.normpath( file ) 116 132 d = os.path.dirname ( p ) 117 sh( 'mkdir', '-p', d, output =subprocess.DEVNULL, error=subprocess.DEVNULL )133 sh( 'mkdir', '-p', d, output_file=subprocess.DEVNULL, error=subprocess.DEVNULL ) 118 134 119 135 … … 138 154 lhs, 139 155 rhs, 140 output =subprocess.PIPE156 output_file=subprocess.PIPE 141 157 ) 142 158 143 159 # call make 144 def make(target, *, flags = '', output = None, error = None, error_file = None, silent = False):160 def make(target, *, flags = '', output_file = None, error = None, error_file = None, silent = False): 145 161 test_param = """test="%s" """ % (error_file) if error_file else None 146 162 cmd = [ … … 151 167 settings.debug.flags, 152 168 settings.install.flags, 169 settings.distcc if settings.distribute else None, 153 170 flags, 154 171 target 155 172 ] 156 173 cmd = [s for s in cmd if s] 157 return sh(*cmd, output =output, error=error)174 return sh(*cmd, output_file=output_file, error=error) 158 175 159 176 def which(program): 160 fpath, fname = os.path.split(program) 161 if fpath: 162 if is_exe(program): 163 return program 164 else: 165 for path in os.environ["PATH"].split(os.pathsep): 166 exe_file = os.path.join(path, program) 167 if is_exe(exe_file): 168 return exe_file 169 170 return None 177 fpath, fname = os.path.split(program) 178 if fpath: 179 if is_exe(program): 180 return program 181 else: 182 for path in os.environ["PATH"].split(os.pathsep): 183 exe_file = os.path.join(path, program) 184 if is_exe(exe_file): 185 return exe_file 186 return None 171 187 172 188 @contextlib.contextmanager … … 201 217 # cat one file into the other 202 218 def cat(source, dest): 203 ret, _ = sh("cat", source, output =dest)219 ret, _ = sh("cat", source, output_file=dest) 204 220 return ret 205 221 … … 256 272 os.write(int(make_jobs_fds.group(3)), tokens) 257 273 else : 258 options.jobs = multiprocessing.cpu_count() 274 if settings.distribute: 275 ret, jstr = sh("distcc", "-j", output_file=subprocess.PIPE, ignore_dry_run=True) 276 if ret == 0: 277 options.jobs = int(jstr.strip()) 278 else : 279 options.jobs = multiprocessing.cpu_count() 280 else: 281 options.jobs = multiprocessing.cpu_count() 259 282 else : 260 283 force = True … … 274 297 ################################################################################ 275 298 299 # get hash for given configuration 300 def config_hash(): 301 path = os.path.normpath(os.path.join( 302 settings.SRCDIR, 303 )) 304 305 distcc_hash = os.path.join(settings.SRCDIR, '../tools/build/distcc_hash') 306 config = "%s-%s" % (settings.arch.target, settings.debug.path) 307 _, out = sh(distcc_hash, config, output_file=subprocess.PIPE, ignore_dry_run=True) 308 return out.strip() 309 310 # get pretty string for time of day 276 311 def pretty_now(): 277 312 ts = time.time() … … 308 343 return 1, "ERR No core dump" 309 344 310 return sh('gdb', '-n', path, core, '-batch', '-x', cmd, output =subprocess.PIPE)345 return sh('gdb', '-n', path, core, '-batch', '-x', cmd, output_file=subprocess.PIPE) 311 346 312 347 def core_archive(dst, name, exe): 313 # Get the files to copy348 # Get the core dump 314 349 core = os.path.join(os.getcwd(), "core" ) 315 350 316 # Uncomment if we want timestamps on coredumps317 # dst = os.path.join(dst, "%s_%s" % (name, pretty_now()))351 # update the path for this test 352 dst = os.path.join(dst, name) 318 353 319 354 # make a directory for this test 320 mkdir(os.path.join(dst, "dir")) 355 # mkdir makes the parent directory only so add a dummy 356 mkdir(os.path.join(dst, name )) 321 357 322 358 # moves the files … … 328 364 329 365 class Timed: 330 331 332 333 334 335 336 366 def __enter__(self): 367 self.start = time.time() 368 return self 369 370 def __exit__(self, *args): 371 self.end = time.time() 372 self.duration = self.end - self.start 337 373 338 374 def timed(src, timeout): 339 375 expire = time.time() + timeout 340 376 i = iter(src) 341 while True: 342 yield i.next(max(expire - time.time(), 0)) 377 with contextlib.suppress(StopIteration): 378 while True: 379 yield i.next(max(expire - time.time(), 0))
Note:
See TracChangeset
for help on using the changeset viewer.
