source: tests/pybin/tools.py@ fe610ab

ADT ast-experimental enum pthread-emulation qualifiedEnum
Last change on this file since fe610ab was a83012bf, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Missing a character in last push.

  • Property mode set to 100644
File size: 11.5 KB
RevLine 
[c07d724]1import __main__
2import argparse
[a45fc7b]3import contextlib
[dcfedca]4import datetime
[0c13238]5import fileinput
[bacc36c]6import multiprocessing
[c07d724]7import os
8import re
[0c13238]9import resource
[bacc36c]10import signal
[c07d724]11import stat
[1bb2488]12import subprocess
[bacc36c]13import sys
[f806b61]14import tempfile
[0c13238]15import time
[5b993e0]16import types
[c07d724]17
[bacc36c]18from pybin import settings
[c07d724]19
[bacc36c]20################################################################################
21# shell helpers
22################################################################################
23
[c07d724]24# helper functions to run terminal commands
[eea20cd]25def sh(*cmd, timeout = False, output_file = None, input_file = None, input_text = None, error = subprocess.STDOUT, ignore_dry_run = False):
[8364209]26 try:
27 cmd = list(cmd)
[1bb2488]28
[8364209]29 if input_file and input_text:
30 return 401, "Cannot use both text and file inputs"
[d65f92c]31
[8364209]32 # if this is a dry_run, only print the commands that would be ran
33 if settings.dry_run and not ignore_dry_run:
34 cmd = "{} cmd: {}".format(os.getcwd(), ' '.join(cmd))
35 if output_file and not isinstance(output_file, int):
36 cmd += " > "
37 cmd += output_file
[ea62265]38
[8364209]39 if error and not isinstance(error, int):
40 cmd += " 2> "
41 cmd += error
[ea62265]42
[8364209]43 if input_file and not isinstance(input_file, int) and os.path.isfile(input_file):
44 cmd += " < "
45 cmd += input_file
[ea62265]46
[8364209]47 print(cmd)
48 return 0, None
[bacc36c]49
[8364209]50 with contextlib.ExitStack() as onexit:
51 # add input redirection if needed
52 input_file = openfd(input_file, 'r', onexit, True)
[a45fc7b]53
[8364209]54 # add output redirection if needed
55 output_file = openfd(output_file, 'w', onexit, False)
[f806b61]56
[8364209]57 # add error redirection if needed
58 error = openfd(error, 'w', onexit, False)
[a45fc7b]59
[8364209]60 # run the desired command
61 # use with statement to make sure proc is cleaned
62 # don't use subprocess.run because we want to send SIGABRT on exit
63 with subprocess.Popen(
[2b10f95]64 cmd,
[d65f92c]65 **({'input' : bytes(input_text, encoding='utf-8')} if input_text else {'stdin' : input_file}),
66 stdout = output_file,
[8364209]67 stderr = error
68 ) as proc:
69
70 try:
71 out, _ = proc.communicate(
72 timeout = settings.timeout.single if timeout else None
73 )
74
[080b0a1]75 return proc.returncode, out.decode("latin-1") if out else None
[8364209]76 except subprocess.TimeoutExpired:
[d658183]77 if settings.timeout2gdb:
78 print("Process {} timeout".format(proc.pid))
79 proc.communicate()
80 return 124, str(None)
81 else:
82 proc.send_signal(signal.SIGABRT)
83 proc.communicate()
84 return 124, str(None)
[8364209]85
86 except Exception as ex:
87 print ("Unexpected error: %s" % ex)
88 raise
[c07d724]89
[f866d15]90def is_empty(fname):
91 if not os.path.isfile(fname):
92 return True
93
94 if os.stat(fname).st_size == 0:
95 return True
96
97 return False
98
[f85bc15]99def is_ascii(fname):
[202ad72]100 if settings.dry_run:
101 print("is_ascii: %s" % fname)
[f866d15]102 return (True, "")
[202ad72]103
[f85bc15]104 if not os.path.isfile(fname):
[f866d15]105 return (False, "No file")
[f85bc15]106
[f866d15]107 code, out = sh("file", fname, output_file=subprocess.PIPE)
[f85bc15]108 if code != 0:
[f866d15]109 return (False, "'file EXPECT' failed with code {}".format(code))
[f85bc15]110
111 match = re.search(".*: (.*)", out)
112
113 if not match:
[f866d15]114 return (False, "Unreadable file type: '{}'".format(out))
115
116 if "ASCII text" in match.group(1):
117 return (True, "")
[f85bc15]118
[f866d15]119 return (False, "File type should be 'ASCII text', was '{}'".format(match.group(1)))
[f85bc15]120
[5bf1f3e]121def is_exe(fname):
122 return os.path.isfile(fname) and os.access(fname, os.X_OK)
123
[f806b61]124def openfd(file, mode, exitstack, checkfile):
125 if not file:
126 return file
127
128 if isinstance(file, int):
129 return file
130
131 if checkfile and not os.path.isfile(file):
132 return None
133
[09bbe78]134 file = open(file, mode, encoding="latin-1") # use latin-1 so all chars mean something.
[f806b61]135 exitstack.push(file)
136 return file
137
[c07d724]138# Remove 1 or more files silently
[bacc36c]139def rm( files ):
[5b993e0]140 if isinstance(files, str ): files = [ files ]
141 for file in files:
[d65f92c]142 sh( 'rm', '-f', file, output_file=subprocess.DEVNULL, error=subprocess.DEVNULL )
[c07d724]143
[a95c117]144# Create 1 or more directory
145def mkdir( files ):
[5b993e0]146 if isinstance(files, str ): files = [ files ]
147 for file in files:
148 p = os.path.normpath( file )
149 d = os.path.dirname ( p )
[d65f92c]150 sh( 'mkdir', '-p', d, output_file=subprocess.DEVNULL, error=subprocess.DEVNULL )
[28582b2]151
[a95c117]152
[c07d724]153def chdir( dest = __main__.__file__ ):
154 abspath = os.path.abspath(dest)
155 dname = os.path.dirname(abspath)
156 os.chdir(dname)
157
[bacc36c]158# diff two files
159def diff( lhs, rhs ):
160 # fetch return code and error from the diff command
[a45fc7b]161 return sh(
162 '''diff''',
163 '''--text''',
164 '''--old-group-format=\t\tmissing lines :\n%<''',
165 '''--new-line-format=\t\t%dn\t%L''',
166 '''--new-group-format=\t\tnew lines : \n%>''',
167 '''--old-line-format=\t\t%dn\t%L''',
168 '''--unchanged-group-format=%=''',
169 '''--changed-group-format=\t\texpected :\n%<\t\tgot :\n%>''',
170 '''--unchanged-line-format=''',
171 lhs,
172 rhs,
[d65f92c]173 output_file=subprocess.PIPE
[a45fc7b]174 )
[bacc36c]175
176# call make
[d65f92c]177def make(target, *, flags = '', output_file = None, error = None, error_file = None, silent = False):
[a45fc7b]178 test_param = """test="%s" """ % (error_file) if error_file else None
179 cmd = [
180 *settings.make,
181 '-s' if silent else None,
[bacc36c]182 test_param,
[99581ee]183 settings.ast.flags,
[575a6e5]184 settings.arch.flags,
[f3b9efc]185 settings.debug.flags,
[a5121bf]186 settings.install.flags,
[d65f92c]187 settings.distcc if settings.distribute else None,
[bacc36c]188 flags,
[a45fc7b]189 target
190 ]
191 cmd = [s for s in cmd if s]
[d65f92c]192 return sh(*cmd, output_file=output_file, error=error)
[bacc36c]193
[a468e1e9]194def make_recon(target):
195 cmd = [
196 *settings.make,
197 '-W',
198 os.path.abspath(os.path.join(settings.BUILDDIR, '../driver/cfa')),
199 '--recon',
200 target
201 ]
202 cmd = [s for s in cmd if s]
203 return sh(*cmd, output_file=subprocess.PIPE)
204
[ed45af6]205def which(program):
[0f5da65]206 fpath, fname = os.path.split(program)
207 if fpath:
208 if is_exe(program):
209 return program
210 else:
211 for path in os.environ["PATH"].split(os.pathsep):
212 exe_file = os.path.join(path, program)
213 if is_exe(exe_file):
214 return exe_file
215 return None
[0c13238]216
[f806b61]217@contextlib.contextmanager
218def tempdir():
219 cwd = os.getcwd()
220 with tempfile.TemporaryDirectory() as temp:
221 os.chdir(temp)
222 try:
223 yield temp
224 finally:
225 os.chdir(cwd)
226
[35a408b7]227def killgroup():
228 try:
229 os.killpg(os.getpgrp(), signal.SIGINT)
230 except KeyboardInterrupt:
231 pass # expected
232 except Exception as exc:
233 print("Unexpected exception", file=sys.stderr)
234 print(exc, file=sys.stderr)
235 sys.stderr.flush()
236 sys.exit(2)
237
[bacc36c]238################################################################################
239# file handling
240################################################################################
[0c13238]241# move a file
242def mv(source, dest):
[a45fc7b]243 ret, _ = sh("mv", source, dest)
[0c13238]244 return ret
245
246# cat one file into the other
247def cat(source, dest):
[d65f92c]248 ret, _ = sh("cat", source, output_file=dest)
[0c13238]249 return ret
[bacc36c]250
[c07d724]251# helper function to replace patterns in a file
252def file_replace(fname, pat, s_after):
[202ad72]253 if settings.dry_run:
254 print("replacing '%s' with '%s' in %s" % (pat, s_after, fname))
255 return
256
[f85bc15]257 file = fileinput.FileInput(fname, inplace=True, backup='.bak')
258 for line in file:
259 print(line.replace(pat, s_after), end='')
260 file.close()
[c07d724]261
[0ad0c55]262# helper function to check if a files contains only a specific string
[5bf1f3e]263def file_contains_only(file, text) :
[eb67b47]264 with open(file, encoding="latin-1") as f: # use latin-1 so all chars mean something.
[c07d724]265 ff = f.read().strip()
266 result = ff == text.strip()
267
[5bf1f3e]268 return result
[c07d724]269
[bacc36c]270# transform path to canonical form
[5bf1f3e]271def canonical_path(path):
[8e516fd]272 abspath = os.path.abspath(os.path.realpath(__main__.__file__))
[f85bc15]273 dname = os.path.dirname(abspath)
274 return os.path.join(dname, os.path.normpath(path) )
[c07d724]275
[bacc36c]276# compare path even if form is different
[5bf1f3e]277def path_cmp(lhs, rhs):
278 return canonical_path( lhs ) == canonical_path( rhs )
[c07d724]279
[bacc36c]280# walk all files in a path
[5bf1f3e]281def path_walk( op ):
[56de5932]282 dname = settings.SRCDIR
[5b993e0]283 for dirname, _, names in os.walk(dname):
284 for name in names:
285 path = os.path.join(dirname, name)
286 op( path )
[bacc36c]287
288################################################################################
289# system
290################################################################################
291# count number of jobs to create
[5bf1f3e]292def job_count( options, tests ):
[bacc36c]293 # check if the user already passed in a number of jobs for multi-threading
[d142ec5]294 if not options.jobs:
295 make_flags = os.environ.get('MAKEFLAGS')
296 force = bool(make_flags)
297 make_jobs_fds = re.search("--jobserver-(auth|fds)=\s*([0-9]+),([0-9]+)", make_flags) if make_flags else None
298 if make_jobs_fds :
299 tokens = os.read(int(make_jobs_fds.group(2)), 1024)
300 options.jobs = len(tokens)
301 os.write(int(make_jobs_fds.group(3)), tokens)
302 else :
[34e1494]303 if settings.distribute:
304 ret, jstr = sh("distcc", "-j", output_file=subprocess.PIPE, ignore_dry_run=True)
305 if ret == 0:
306 options.jobs = int(jstr.strip())
307 else :
308 options.jobs = multiprocessing.cpu_count()
309 else:
310 options.jobs = multiprocessing.cpu_count()
[bacc36c]311 else :
[d142ec5]312 force = True
[bacc36c]313
314 # make sure we have a valid number of jobs that corresponds to user input
315 if options.jobs <= 0 :
316 print('ERROR: Invalid number of jobs', file=sys.stderr)
317 sys.exit(1)
318
[d142ec5]319 return min( options.jobs, len(tests) ), force
[bacc36c]320
[0c13238]321# enable core dumps for all the test children
322resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
323
[bacc36c]324################################################################################
325# misc
326################################################################################
327
[d65f92c]328# get hash for given configuration
329def config_hash():
330 path = os.path.normpath(os.path.join(
331 settings.SRCDIR,
332 ))
333
334 distcc_hash = os.path.join(settings.SRCDIR, '../tools/build/distcc_hash')
335 config = "%s-%s" % (settings.arch.target, settings.debug.path)
[eea20cd]336 _, out = sh(distcc_hash, config, output_file=subprocess.PIPE, ignore_dry_run=True)
[d65f92c]337 return out.strip()
338
[5c4a473]339# get pretty string for time of day
[dcfedca]340def pretty_now():
341 ts = time.time()
342 print(ts, file=sys.stderr)
343 return datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H:%M:%S')
344
[bacc36c]345# check if arguments is yes or no
346def yes_no(string):
347 if string == "yes" :
348 return True
349 if string == "no" :
350 return False
351 raise argparse.ArgumentTypeError(msg)
[f3b9efc]352
[136f86b]353# Convert a function that converts a string to one that converts comma separated string.
354def comma_separated(elements):
355 return lambda string: [elements(part) for part in string.split(',')]
356
[ed45af6]357def fancy_print(text):
358 column = which('column')
359 if column:
[1bb2488]360 subprocess.run(column, input=bytes(text + "\n", "UTF-8"))
[ed45af6]361 else:
362 print(text)
[0c13238]363
364
[5bf1f3e]365def core_info(path):
[f806b61]366 if not os.path.isfile(path):
367 return 1, "ERR Executable path is wrong"
368
[0c13238]369 cmd = os.path.join(settings.SRCDIR, "pybin/print-core.gdb")
370 if not os.path.isfile(cmd):
371 return 1, "ERR Printing format for core dumps not found"
372
[f806b61]373 core = os.path.join(os.getcwd(), "core" )
[0c13238]374
375 if not os.path.isfile(core):
[a83012bf]376 return 1, "ERR No core dump (limit soft: {} hard: {})".format(*resource.getrlimit(resource.RLIMIT_CORE))
[0c13238]377
[22a4292]378 try:
379 return sh('gdb', '-n', path, core, '-batch', '-x', cmd, output_file=subprocess.PIPE)
380 except:
381 return 1, "ERR Could not read core with gdb"
[0c13238]382
[dcfedca]383def core_archive(dst, name, exe):
[143e6f3]384 # Get the core dump
[dcfedca]385 core = os.path.join(os.getcwd(), "core" )
386
[143e6f3]387 # update the path for this test
388 dst = os.path.join(dst, name)
[dcfedca]389
390 # make a directory for this test
[143e6f3]391 # mkdir makes the parent directory only so add a dummy
[8c28967]392 mkdir( os.path.join(dst, "core") )
[dcfedca]393
394 # moves the files
395 mv( core, os.path.join(dst, "core" ) )
[8c28967]396 mv( exe , os.path.join(dst, "exe" ) )
[dcfedca]397
398 # return explanatory test
399 return "Archiving %s (executable and core) to %s" % (os.path.relpath(exe, settings.BUILDDIR), os.path.relpath(dst, settings.original_path))
400
[0c13238]401class Timed:
[0f5da65]402 def __enter__(self):
403 self.start = time.time()
404 return self
[0c13238]405
[0f5da65]406 def __exit__(self, *args):
407 self.end = time.time()
408 self.duration = self.end - self.start
[35a408b7]409
410def timed(src, timeout):
411 expire = time.time() + timeout
412 i = iter(src)
[0f5da65]413 with contextlib.suppress(StopIteration):
414 while True:
415 yield i.next(max(expire - time.time(), 0))
[76de075]416
417def fmtDur( duration ):
418 if duration :
419 hours, rem = divmod(duration, 3600)
420 minutes, rem = divmod(rem, 60)
421 seconds, millis = divmod(rem, 1)
422 return "%2d:%02d.%03d" % (minutes, seconds, millis * 1000)
423 return " n/a"
Note: See TracBrowser for help on using the repository browser.