Changeset 510e6f9 for tests

Timestamp:

Mar 11, 2022, 1:56:07 PM (4 years ago)

Author:

caparsons <caparson@…>

Branches:

ADT, ast-experimental, enum, master, pthread-emulation, qualifiedEnum

Children:

623d1c8

Parents:

eb3bc52 (diff), 630c4bb (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

Location:

tests

Files:

• Makefile.am (modified) (2 diffs)
• concurrent/.expect/keywordErrors.nast.txt (added)
• concurrent/.expect/keywordErrors.oast.txt (moved) (moved from tests/concurrent/.expect/keywordErrors.txt )
• concurrent/.expect/mainError.nast.txt (added)
• concurrent/.expect/mainError.oast.txt (added)
• concurrent/mainError.cfa (added)
• io/.expect/away_fair.txt (added)
• io/away_fair.cfa (added)
• io/many_read.cfa (modified) (1 diff)
• pybin/settings.py (modified) (4 diffs)
• pybin/tools.py (modified) (8 diffs)
• test.py (modified) (9 diffs)

tests/Makefile.am

@@ -66 +66 @@
 PRETTY_PATH=mkdir -p $(dir $(abspath ${@})) && cd ${srcdir} &&
 
-.PHONY: list .validate
-.INTERMEDIATE: .validate .validate.cfa
+.PHONY: list .validate .test_makeflags
+.INTERMEDIATE: .validate .validate.cfa .test_makeflags
 EXTRA_PROGRAMS = avl_test linkonce .dummy_hack # build but do not install
 EXTRA_DIST = test.py \
@@ -123 +123 @@
         @+${TEST_PY} --list ${concurrent}
 
+.test_makeflags:
+        @echo "${MAKEFLAGS}"
+
 .validate: .validate.cfa
         $(CFACOMPILE) .validate.cfa -fsyntax-only -Wall -Wextra -Werror

tests/io/many_read.cfa

@@ -5 +5 @@
 // file "LICENCE" distributed with Cforall.
 //
-// many_read.cfa -- Make sure that multiple concurrent reads to mess up.
+// many_read.cfa -- Make sure that multiple concurrent reads don't mess up.
 //
 // Author           : Thierry Delisle

tests/pybin/settings.py

@@ -155 +155 @@
         global generating
         global make
+        global make_jobfds
         global output_width
         global timeout
@@ -168 +169 @@
         generating   = options.regenerate_expected
         make         = ['make']
+        make_jobfds  = []
         output_width = 24
         timeout      = Timeouts(options.timeout, options.global_timeout)
@@ -177 +179 @@
                 os.putenv('DISTCC_LOG', os.path.join(BUILDDIR, 'distcc_error.log'))
 
-def update_make_cmd(force, jobs):
+def update_make_cmd(flags):
         global make
-
-        make = ['make'] if not force else ['make', "-j%i" % jobs]
+        make = ['make', *flags]
+
+def update_make_fds(r, w):
+        global make_jobfds
+        make_jobfds = (r, w)
 
 def validate():
@@ -187 +192 @@
         global distcc
         distcc       = "DISTCC_CFA_PATH=~/.cfadistcc/%s/cfa" % tools.config_hash()
-        errf = os.path.join(BUILDDIR, ".validate.err")
-        make_ret, out = tools.make( ".validate", error_file = errf, output_file=subprocess.DEVNULL, error=subprocess.DEVNULL )
+        make_ret, out, err = tools.make( ".validate", output_file=subprocess.PIPE, error=subprocess.PIPE )
         if make_ret != 0:
-                with open (errf, "r") as myfile:
-                        error=myfile.read()
                 print("ERROR: Invalid configuration %s:%s" % (arch.string, debug.string), file=sys.stderr)
-                print("       verify returned : \n%s" % error, file=sys.stderr)
-                tools.rm(errf)
+                print("       verify returned : \n%s" % err, file=sys.stderr)
                 sys.exit(1)
-
-        tools.rm(errf)
 
 def prep_output(tests):

tests/pybin/tools.py

@@ -23 +23 @@
 
 # helper functions to run terminal commands
-def sh(*cmd, timeout = False, output_file = None, input_file = None, input_text = None, error = subprocess.STDOUT, ignore_dry_run = False):
+def sh(*cmd, timeout = False, output_file = None, input_file = None, input_text = None, error = subprocess.STDOUT, ignore_dry_run = False, pass_fds = []):
         try:
                 cmd = list(cmd)
@@ -65 +65 @@
                                 **({'input' : bytes(input_text, encoding='utf-8')} if input_text else {'stdin' : input_file}),
                                 stdout  = output_file,
-                                stderr  = error
+                                stderr  = error,
+                                pass_fds = pass_fds
                         ) as proc:
 
                                 try:
-                                        out, _ = proc.communicate(
+                                        out, errout = proc.communicate(
                                                 timeout = settings.timeout.single if timeout else None
                                         )
 
-                                        return proc.returncode, out.decode("latin-1") if out else None
+                                        return proc.returncode, out.decode("latin-1") if out else None, errout.decode("latin-1") if errout else None
                                 except subprocess.TimeoutExpired:
                                         if settings.timeout2gdb:
                                                 print("Process {} timeout".format(proc.pid))
                                                 proc.communicate()
-                                                return 124, str(None)
+                                                return 124, str(None), "Subprocess Timeout 2 gdb"
                                         else:
                                                 proc.send_signal(signal.SIGABRT)
                                                 proc.communicate()
-                                                return 124, str(None)
+                                                return 124, str(None), "Subprocess Timeout 2 gdb"
 
         except Exception as ex:
@@ -105 +106 @@
                 return (False, "No file")
 
-        code, out = sh("file", fname, output_file=subprocess.PIPE)
+        code, out, err = sh("file", fname, output_file=subprocess.PIPE)
         if code != 0:
-                return (False, "'file EXPECT' failed with code {}".format(code))
+                return (False, "'file EXPECT' failed with code {} '{}'".format(code, err))
 
         match = re.search(".*: (.*)", out)
@@ -190 +191 @@
         ]
         cmd = [s for s in cmd if s]
-        return sh(*cmd, output_file=output_file, error=error)
+        return sh(*cmd, output_file=output_file, error=error, pass_fds=settings.make_jobfds)
 
 def make_recon(target):
@@ -241 +242 @@
 # move a file
 def mv(source, dest):
-        ret, _ = sh("mv", source, dest)
+        ret, _, _ = sh("mv", source, dest)
         return ret
 
 # cat one file into the other
 def cat(source, dest):
-        ret, _ = sh("cat", source, output_file=dest)
+        ret, _, _ = sh("cat", source, output_file=dest)
         return ret
 
@@ -289 +290 @@
 #               system
 ################################################################################
+def jobserver_version():
+        make_ret, out, err = sh('make', '.test_makeflags', '-j2', output_file=subprocess.PIPE, error=subprocess.PIPE)
+        if make_ret != 0:
+                print("ERROR: cannot find Makefile jobserver version", file=sys.stderr)
+                print("       test returned : {} '{}'".format(make_ret, err), file=sys.stderr)
+                sys.exit(1)
+
+        re_jobs = re.search("--jobserver-(auth|fds)", out)
+        if not re_jobs:
+                print("ERROR: cannot find Makefile jobserver version", file=sys.stderr)
+                print("       MAKEFLAGS are : '{}'".format(out), file=sys.stderr)
+                sys.exit(1)
+
+        return "--jobserver-{}".format(re_jobs.group(1))
+
+def prep_recursive_make(N):
+        if N < 2:
+                return []
+
+        # create the pipe
+        (r, w) = os.pipe()
+
+        # feel it with N-1 tokens, (Why N-1 and not N, I don't know it's in the manpage for make)
+        os.write(w, b'+' * (N - 1));
+
+        # prep the flags for make
+        make_flags = ["-j{}".format(N), "--jobserver-auth={},{}".format(r, w)]
+
+        # tell make about the pipes
+        os.environ["MAKEFLAGS"] = os.environ["MFLAGS"] = " ".join(make_flags)
+
+        # make sure pass the pipes to our children
+        settings.update_make_fds(r, w)
+
+        return make_flags
+
+def prep_unlimited_recursive_make():
+        # prep the flags for make
+        make_flags = ["-j"]
+
+        # tell make about the pipes
+        os.environ["MAKEFLAGS"] = os.environ["MFLAGS"] = "-j"
+
+        return make_flags
+
+
+def eval_hardware():
+        # we can create as many things as we want
+        # how much hardware do we have?
+        if settings.distribute:
+                # remote hardware is allowed
+                # how much do we have?
+                ret, jstr, _ = sh("distcc", "-j", output_file=subprocess.PIPE, ignore_dry_run=True)
+                return int(jstr.strip()) if ret == 0 else multiprocessing.cpu_count()
+        else:
+                # remote isn't allowed, use local cpus
+                return multiprocessing.cpu_count()
+
 # count number of jobs to create
-def job_count( options, tests ):
+def job_count( options ):
         # check if the user already passed in a number of jobs for multi-threading
-        if not options.jobs:
-                make_flags = os.environ.get('MAKEFLAGS')
-                force = bool(make_flags)
-                make_jobs_fds = re.search("--jobserver-(auth|fds)=\s*([0-9]+),([0-9]+)", make_flags) if make_flags else None
-                if make_jobs_fds :
-                        tokens = os.read(int(make_jobs_fds.group(2)), 1024)
-                        options.jobs = len(tokens)
-                        os.write(int(make_jobs_fds.group(3)), tokens)
-                else :
-                        if settings.distribute:
-                                ret, jstr = sh("distcc", "-j", output_file=subprocess.PIPE, ignore_dry_run=True)
-                                if ret == 0:
-                                        options.jobs = int(jstr.strip())
-                                else :
-                                        options.jobs = multiprocessing.cpu_count()
-                        else:
-                                options.jobs = multiprocessing.cpu_count()
+        make_env = os.environ.get('MAKEFLAGS')
+        make_flags = make_env.split() if make_env else None
+        jobstr = jobserver_version()
+
+        if options.jobs and make_flags:
+                print('WARNING: -j options should not be specified when called form Make', file=sys.stderr)
+
+        # Top level make is calling the shots, just follow
+        if make_flags:
+                # do we have -j and --jobserver-...
+                jobopt = None
+                exists_fds = None
+                for f in make_flags:
+                        jobopt = f if f.startswith("-j") else jobopt
+                        exists_fds = f if f.startswith(jobstr) else exists_fds
+
+                # do we have limited parallelism?
+                if exists_fds :
+                        try:
+                                rfd, wfd = tuple(exists_fds.split('=')[1].split(','))
+                        except:
+                                print("ERROR: jobserver has unrecoginzable format, was '{}'".format(exists_fds), file=sys.stderr)
+                                sys.exit(1)
+
+                        # read the token pipe to count number of available tokens and restore the pipe
+                        # this assumes the test suite script isn't invoked in parellel with something else
+                        tokens = os.read(int(rfd), 65536)
+                        os.write(int(wfd), tokens)
+
+                        # the number of tokens is off by one for obscure but well documented reason
+                        # see man make for more details
+                        options.jobs = len(tokens) + 1
+
+                # do we have unlimited parallelism?
+                elif jobopt and jobopt != "-j1":
+                        # check that this actually make sense
+                        if jobopt != "-j":
+                                print("ERROR: -j option passed by make but no {}, was '{}'".format(jobstr, jobopt), file=sys.stderr)
+                                sys.exit(1)
+
+                        options.jobs = eval_hardware()
+                        flags = prep_unlimited_recursive_make()
+
+
+                # then no parallelism
+                else:
+                        options.jobs = 1
+
+                # keep all flags make passed along, except the weird 'w' which is about subdirectories
+                flags = [f for f in make_flags if f != 'w']
+
+        # Arguments are calling the shots, fake the top level make
+        elif options.jobs :
+
+                # make sure we have a valid number of jobs that corresponds to user input
+                if options.jobs < 0 :
+                        print('ERROR: Invalid number of jobs', file=sys.stderr)
+                        sys.exit(1)
+
+                flags = prep_recursive_make(options.jobs)
+
+        # Arguments are calling the shots, fake the top level make, but 0 is a special case
+        elif options.jobs == 0:
+                options.jobs = eval_hardware()
+                flags = prep_unlimited_recursive_make()
+
+        # No one says to run in parallel, then don't
         else :
-                force = True
-
-        # make sure we have a valid number of jobs that corresponds to user input
-        if options.jobs <= 0 :
-                print('ERROR: Invalid number of jobs', file=sys.stderr)
-                sys.exit(1)
-
-        return min( options.jobs, len(tests) ), force
+                options.jobs = 1
+                flags = []
+
+        # Make sure we call make as expected
+        settings.update_make_cmd( flags )
+
+        # return the job count
+        return options.jobs
 
 # enable core dumps for all the test children
@@ -334 +443 @@
         distcc_hash = os.path.join(settings.SRCDIR, '../tools/build/distcc_hash')
         config = "%s-%s" % (settings.arch.target, settings.debug.path)
-        _, out = sh(distcc_hash, config, output_file=subprocess.PIPE, ignore_dry_run=True)
+        _, out, _ = sh(distcc_hash, config, output_file=subprocess.PIPE, ignore_dry_run=True)
         return out.strip()
 
@@ -374 +483 @@
 
         if not os.path.isfile(core):
-                return 1, "ERR No core dump (limit soft: {} hard: {})".format(*resource.getrlimit(resource.RLIMIT_CORE))
+                return 1, "ERR No core dump, expected '{}' (limit soft: {} hard: {})".format(core, *resource.getrlimit(resource.RLIMIT_CORE))
 
         try:
-                return sh('gdb', '-n', path, core, '-batch', '-x', cmd, output_file=subprocess.PIPE)
+                ret, out, err = sh('gdb', '-n', path, core, '-batch', '-x', cmd, output_file=subprocess.PIPE)
+                if ret == 0:
+                        return 0, out
+                else:
+                        return 1, err
         except:
                 return 1, "ERR Could not read core with gdb"

tests/test.py

@@ -140 +140 @@
         parser.add_argument('--regenerate-expected', help='Regenerate the .expect by running the specified tets, can be used with --all option', action='store_true')
         parser.add_argument('--archive-errors', help='If called with a valid path, on test crashes the test script will copy the core dump and the executable to the specified path.', type=str, default='')
-        parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int)
+        parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously, 0 (default) for unlimited', nargs='?', const=0, type=int)
         parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
         parser.add_argument('--list-dist', help='List all tests for distribution', action='store_true')
@@ -195 +195 @@
         # build, skipping to next test on error
         with Timed() as comp_dur:
-                make_ret, _ = make( test.target(), output_file=subprocess.DEVNULL, error=out_file, error_file = err_file )
+                make_ret, _, _ = make( test.target(), output_file=subprocess.DEVNULL, error=out_file, error_file = err_file )
 
         # ----------
@@ -208 +208 @@
                                 if settings.dry_run or is_exe(exe_file):
                                         # run test
-                                        retcode, _ = sh(exe_file, output_file=out_file, input_file=in_file, timeout=True)
+                                        retcode, _, _ = sh(exe_file, output_file=out_file, input_file=in_file, timeout=True)
                                 else :
                                         # simply cat the result into the output
@@ -226 +226 @@
                         else :
                                 # fetch return code and error from the diff command
-                                retcode, error = diff(cmp_file, out_file)
+                                retcode, error, _ = diff(cmp_file, out_file)
 
                 else:
@@ -366 +366 @@
                         print(os.path.relpath(t.expect(), settings.SRCDIR), end=' ')
                         print(os.path.relpath(t.input() , settings.SRCDIR), end=' ')
-                        code, out = make_recon(t.target())
+                        code, out, err = make_recon(t.target())
 
                         if code != 0:
-                                print('ERROR: recond failed for test {}'.format(t.target()), file=sys.stderr)
+                                print('ERROR: recond failed for test {}: {} \'{}\''.format(t.target(), code, err), file=sys.stderr)
                                 sys.exit(1)
 
@@ -417 +417 @@
                         if is_empty(t.expect()):
                                 print('WARNING: test "{}" has empty .expect file'.format(t.target()), file=sys.stderr)
+
+        options.jobs = job_count( options )
 
         # for each build configurations, run the test
@@ -430 +432 @@
                         local_tests = settings.ast.filter( tests )
                         local_tests = settings.arch.filter( local_tests )
-                        options.jobs, forceJobs = job_count( options, local_tests )
-                        settings.update_make_cmd(forceJobs, options.jobs)
 
                         # check the build configuration works
                         settings.validate()
+                        jobs = min(options.jobs, len(local_tests))
 
                         # print configuration
@@ -440 +441 @@
                                 'Regenerating' if settings.generating else 'Running',
                                 len(local_tests),
-                                options.jobs,
+                                jobs,
                                 settings.ast.string,
                                 settings.arch.string,
@@ -450 +451 @@
 
                         # otherwise run all tests and make sure to return the correct error code
-                        failed = run_tests(local_tests, options.jobs)
+                        failed = run_tests(local_tests, jobs)
                         if failed:
                                 if not settings.continue_: