Changes in / [86ad276:5da9d6a]


Ignore:
Location:
src/tests
Files:
66 added
68 deleted
4 edited

Legend:

Unmodified
Added
Removed

  • src/tests/Makefile.am

    r86ad276 r5da9d6a  
    2020
    2121if BUILD_CONCURRENCY
    22 concurrent=
     22concurrent = yes
     23quick_test += coroutine thread monitor
     24concurrent_test =               \
     25        coroutine               \
     26        fmtLines                \
     27        pingpong                \
     28        prodcons                \
     29        thread                  \
     30        matrixSum               \
     31        monitor                 \
     32        multi-monitor           \
     33        boundedBuffer           \
     34        preempt                 \
     35        sched-int-block         \
     36        sched-int-disjoint      \
     37        sched-int-wait          \
     38        sched-ext-barge         \
     39        sched-ext-dtor          \
     40        sched-ext-else          \
     41        sched-ext-parse         \
     42        sched-ext-recurse       \
     43        sched-ext-statment      \
     44        sched-ext-when
    2345else
    24 concurrent='-Econcurrent'
     46concurrent=no
     47concurrent_test=
    2548endif
    2649
    … …  
    3962endif
    4063
    41 TEST_FLAGS = $(if $(test), 2> $(test), )
     64TEST_FLAGS = $(if $(test), 2> .err/${@}.log, )
    4265AM_CFLAGS = ${TEST_FLAGS} ${BUILD_FLAGS}
    4366CC = @CFA_BINDIR@/@CFA_NAME@
    … …  
    4770
    4871fstream_test_SOURCES = fstream_test.c
    49 fstream_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
     72fstream_test_CFLAGS = $(if $(test), 2>> .err/fstream_test.log, ) ${BUILD_FLAGS}
    5073
    5174vector_test_SOURCES = vector/vector_int.c vector/array.c vector/vector_test.c
    52 vector_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
     75vector_test_CFLAGS = $(if $(test), 2>> .err/vector_test.log, ) ${BUILD_FLAGS}
    5376
    5477avl_test_SOURCES = avltree/avl_test.c avltree/avl0.c avltree/avl1.c avltree/avl2.c avltree/avl3.c avltree/avl4.c avltree/avl-private.c
    55 avl_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
     78avl_test_CFLAGS = $(if $(test), 2>> .err/avl_test.log, ) ${BUILD_FLAGS}
    5679
    5780all-local :
    58         @+python test.py --debug=${debug} ${concurrent} ${quick_test}
     81        @+python test.py --debug=${debug} --concurrent=${concurrent} ${quick_test}
    5982
    6083all-tests :
    61         @+python test.py --all --debug=${debug} ${concurrent}           # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
     84        @+python test.py --all --debug=${debug} --concurrent=${concurrent}              # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
    6285
    6386clean-local :
    … …  
    6588
    6689list :
    67         @+python test.py --list ${concurrent}
     90        @+python test.py --list --concurrent=${concurrent}
    6891
    6992concurrency :
    70         @+python test.py --debug=${debug} ${concurrent} ${concurrent_test}
     93        @+python test.py --debug=${debug} --concurrent=${concurrent} ${concurrent_test}
    7194
    7295.dummy : .dummy.c @CFA_BINDIR@/@CFA_NAME@

  • src/tests/Makefile.in

    r86ad276 r5da9d6a  
    9191build_triplet = @build@
    9292host_triplet = @host@
    93 @BUILD_DEBUG_FALSE@am__append_1 = -nodebug
    94 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_FALSE@am__append_2 = -debug
    95 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_TRUE@am__append_3 = ${DEBUG_FLAGS}
     93@BUILD_CONCURRENCY_TRUE@am__append_1 = coroutine thread monitor
     94@BUILD_DEBUG_FALSE@am__append_2 = -nodebug
     95@BUILD_DEBUG_TRUE@@BUILD_RELEASE_FALSE@am__append_3 = -debug
     96@BUILD_DEBUG_TRUE@@BUILD_RELEASE_TRUE@am__append_4 = ${DEBUG_FLAGS}
    9697EXTRA_PROGRAMS = fstream_test$(EXEEXT) vector_test$(EXEEXT) \
    9798        avl_test$(EXEEXT)
    … …  
    313314top_srcdir = @top_srcdir@
    314315debug = yes
    315 quick_test = vector_test avl_test operators numericConstants expression enum array typeof cast dtor-early-exit init_once attributes
    316 @BUILD_CONCURRENCY_FALSE@concurrent = '-Econcurrent'
    317 @BUILD_CONCURRENCY_TRUE@concurrent =
     316quick_test = vector_test avl_test operators numericConstants \
     317        expression enum array typeof cast dtor-early-exit init_once \
     318        attributes $(am__append_1)
     319@BUILD_CONCURRENCY_FALSE@concurrent = no
     320@BUILD_CONCURRENCY_TRUE@concurrent = yes
     321@BUILD_CONCURRENCY_FALSE@concurrent_test =
     322@BUILD_CONCURRENCY_TRUE@concurrent_test = \
     323@BUILD_CONCURRENCY_TRUE@        coroutine               \
     324@BUILD_CONCURRENCY_TRUE@        fmtLines                \
     325@BUILD_CONCURRENCY_TRUE@        pingpong                \
     326@BUILD_CONCURRENCY_TRUE@        prodcons                \
     327@BUILD_CONCURRENCY_TRUE@        thread                  \
     328@BUILD_CONCURRENCY_TRUE@        matrixSum               \
     329@BUILD_CONCURRENCY_TRUE@        monitor                 \
     330@BUILD_CONCURRENCY_TRUE@        multi-monitor           \
     331@BUILD_CONCURRENCY_TRUE@        boundedBuffer           \
     332@BUILD_CONCURRENCY_TRUE@        preempt                 \
     333@BUILD_CONCURRENCY_TRUE@        sched-int-block         \
     334@BUILD_CONCURRENCY_TRUE@        sched-int-disjoint      \
     335@BUILD_CONCURRENCY_TRUE@        sched-int-wait          \
     336@BUILD_CONCURRENCY_TRUE@        sched-ext-barge         \
     337@BUILD_CONCURRENCY_TRUE@        sched-ext-dtor          \
     338@BUILD_CONCURRENCY_TRUE@        sched-ext-else          \
     339@BUILD_CONCURRENCY_TRUE@        sched-ext-parse         \
     340@BUILD_CONCURRENCY_TRUE@        sched-ext-recurse       \
     341@BUILD_CONCURRENCY_TRUE@        sched-ext-statment      \
     342@BUILD_CONCURRENCY_TRUE@        sched-ext-when
     343
    318344
    319345# applies to both programs
    320346DEBUG_FLAGS =
    321347BUILD_FLAGS = -g -Wall -Wno-unused-function -quiet @CFA_FLAGS@ \
    322         $(am__append_1) $(am__append_2) $(am__append_3)
    323 TEST_FLAGS = $(if $(test), 2> $(test), )
     348        $(am__append_2) $(am__append_3) $(am__append_4)
     349TEST_FLAGS = $(if $(test), 2> .err/${@}.log, )
    324350AM_CFLAGS = ${TEST_FLAGS} ${BUILD_FLAGS}
    325351fstream_test_SOURCES = fstream_test.c
    326 fstream_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
     352fstream_test_CFLAGS = $(if $(test), 2>> .err/fstream_test.log, ) ${BUILD_FLAGS}
    327353vector_test_SOURCES = vector/vector_int.c vector/array.c vector/vector_test.c
    328 vector_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
     354vector_test_CFLAGS = $(if $(test), 2>> .err/vector_test.log, ) ${BUILD_FLAGS}
    329355avl_test_SOURCES = avltree/avl_test.c avltree/avl0.c avltree/avl1.c avltree/avl2.c avltree/avl3.c avltree/avl4.c avltree/avl-private.c
    330 avl_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
     356avl_test_CFLAGS = $(if $(test), 2>> .err/avl_test.log, ) ${BUILD_FLAGS}
    331357all: all-am
    332358
    … …  
    807833
    808834all-local :
    809         @+python test.py --debug=${debug} ${concurrent} ${quick_test}
     835        @+python test.py --debug=${debug} --concurrent=${concurrent} ${quick_test}
    810836
    811837all-tests :
    812         @+python test.py --all --debug=${debug} ${concurrent}           # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
     838        @+python test.py --all --debug=${debug} --concurrent=${concurrent}              # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
    813839
    814840clean-local :
    … …  
    816842
    817843list :
    818         @+python test.py --list ${concurrent}
     844        @+python test.py --list --concurrent=${concurrent}
    819845
    820846concurrency :
    821         @+python test.py --debug=${debug} ${concurrent} ${concurrent_test}
     847        @+python test.py --debug=${debug} --concurrent=${concurrent} ${concurrent_test}
    822848
    823849.dummy : .dummy.c @CFA_BINDIR@/@CFA_NAME@

  • src/tests/pybin/tools.py

    r86ad276 r5da9d6a  
    1 from __future__ import print_function
    2 
    31import __main__
    42import argparse
    5 import multiprocessing
    63import os
    74import re
    8 import signal
    95import stat
    10 import sys
    116
    12 from pybin import settings
    137from subprocess import Popen, PIPE, STDOUT
    148
    15 ################################################################################
    16 #               shell helpers
    17 ################################################################################
    18 
    199# helper functions to run terminal commands
    20 def sh(cmd, print2stdout = True, input = None):
    21         # add input redirection if needed
    22         if input and os.path.isfile(input):
    23                 cmd += " < %s" % input
    24 
    25         # if this is a dry_run, only print the commands that would be ran
    26         if settings.dry_run :
     10def sh(cmd, dry_run = False, print2stdout = True):
     11        if dry_run :    # if this is a dry_run, only print the commands that would be ran
    2712                print("cmd: %s" % cmd)
    2813                return 0, None
    29 
    30         # otherwise create a pipe and run the desired command
    31         else :
     14        else :                  # otherwise create a pipe and run the desired command
    3215                proc = Popen(cmd, stdout=None if print2stdout else PIPE, stderr=STDOUT, shell=True)
    3316                out, err = proc.communicate()
    … …  
    3518
    3619# Remove 1 or more files silently
    37 def rm( files ):
     20def rm( files, dry_run = False ):
    3821        try:
    3922                for file in files:
    40                         sh("rm -f %s > /dev/null 2>&1" % file )
     23                        sh("rm -f %s > /dev/null 2>&1" % file, dry_run)
    4124        except TypeError:
    42                 sh("rm -f %s > /dev/null 2>&1" % files )
     25                sh("rm -f %s > /dev/null 2>&1" % files, dry_run)
    4326
    4427def chdir( dest = __main__.__file__ ):
    … …  
    4730        os.chdir(dname)
    4831
     32# helper function to replace patterns in a file
     33def file_replace(fname, pat, s_after):
     34    # first, see if the pattern is even in the file.
     35    with open(fname) as f:
     36        if not any(re.search(pat, line) for line in f):
     37            return # pattern does not occur in file so we are done.
     38
     39    # pattern is in the file, so perform replace operation.
     40    with open(fname) as f:
     41        out_fname = fname + ".tmp"
     42        out = open(out_fname, "w")
     43        for line in f:
     44            out.write(re.sub(pat, s_after, line))
     45        out.close()
     46        os.rename(out_fname, fname)
     47
     48# helper function to check if a files contains only a spacific string
     49def fileContainsOnly(file, text) :
     50        with open(file) as f:
     51                ff = f.read().strip()
     52                result = ff == text.strip()
     53
     54                return result;
     55
     56# check whether or not a file is executable
     57def fileIsExecutable(file) :
     58        try :
     59                fileinfo = os.stat(file)
     60                return bool(fileinfo.st_mode & stat.S_IXUSR)
     61        except Exception as inst:
     62                print(type(inst))    # the exception instance
     63                print(inst.args)     # arguments stored in .args
     64                print(inst)
     65                return False
     66
     67# check if arguments is yes or no
     68def yes_no(string):
     69        if string == "yes" :
     70                return True
     71        if string == "no" :
     72                return False
     73        raise argparse.ArgumentTypeError(msg)
     74        return False
     75
    4976# diff two files
    50 def diff( lhs, rhs ):
     77def diff( lhs, rhs, dry_run ):
    5178        # diff the output of the files
    5279        diff_cmd = ("diff --ignore-all-space "
    … …  
    6794
    6895        # fetch return code and error from the diff command
    69         return sh(diff_cmd % (lhs, rhs), False)
    70 
    71 # call make
    72 def make(target, flags = '', redirects = '', error_file = None, silent = False):
    73         test_param = """test="%s" """ % (error_file) if error_file else ''
    74         cmd = ' '.join([
    75                 settings.make,
    76                 '-s' if silent else '',
    77                 test_param,
    78                 settings.debug.flags,
    79                 flags,
    80                 target,
    81                 redirects
    82         ])
    83         return sh(cmd)
    84 
    85 ################################################################################
    86 #               file handling
    87 ################################################################################
    88 
    89 # helper function to replace patterns in a file
    90 def file_replace(fname, pat, s_after):
    91     # first, see if the pattern is even in the file.
    92     with open(fname) as f:
    93         if not any(re.search(pat, line) for line in f):
    94             return # pattern does not occur in file so we are done.
    95 
    96     # pattern is in the file, so perform replace operation.
    97     with open(fname) as f:
    98         out_fname = fname + ".tmp"
    99         out = open(out_fname, "w")
    100         for line in f:
    101             out.write(re.sub(pat, s_after, line))
    102         out.close()
    103         os.rename(out_fname, fname)
    104 
    105 # helper function to check if a files contains only a specific string
    106 def fileContainsOnly(file, text) :
    107         with open(file) as f:
    108                 ff = f.read().strip()
    109                 result = ff == text.strip()
    110 
    111                 return result;
    112 
    113 # check whether or not a file is executable
    114 def fileIsExecutable(file) :
    115         try :
    116                 fileinfo = os.stat(file)
    117                 return bool(fileinfo.st_mode & stat.S_IXUSR)
    118         except Exception as inst:
    119                 print(type(inst))    # the exception instance
    120                 print(inst.args)     # arguments stored in .args
    121                 print(inst)
    122                 return False
    123 
    124 # transform path to canonical form
    125 def canonicalPath(path):
    126         return os.path.join('.', os.path.normpath(path) )
    127 
    128 # compare path even if form is different
    129 def pathCmp(lhs, rhs):
    130         return canonicalPath( lhs ) == canonicalPath( rhs )
    131 
    132 # walk all files in a path
    133 def pathWalk( op ):
    134         def step(_, dirname, names):
    135                 for name in names:
    136                         path = os.path.join(dirname, name)
    137 
    138                         op( path )
    139 
    140         # Start the walk
    141         os.path.walk('.', step, '')
    142 
    143 ################################################################################
    144 #               system
    145 ################################################################################
    146 
    147 # parses the Makefile to find the machine type (32-bit / 64-bit)
    148 def getMachineType():
    149         sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c')
    150         ret, out = make('.dummy', silent = True)
    151 
    152         if ret != 0:
    153                 print("Failed to identify architecture:")
    154                 print(out)
    155                 print("Stopping")
    156                 rm( (".dummy.c",".dummy") )
    157                 sys.exit(1)
    158 
    159         _, out = sh("file .dummy", print2stdout=False)
    160         rm( (".dummy.c",".dummy") )
    161 
    162         if settings.dry_run :
    163                 return 'x64'
    164 
    165         return re.search(r"[^,]+,([^,]+),", out).group(1).strip()
    166 
    167 # count number of jobs to create
    168 def jobCount( options, tests ):
    169         # check if the user already passed in a number of jobs for multi-threading
    170         make_flags = os.environ.get('MAKEFLAGS')
    171         make_jobs_fds = re.search("--jobserver-(auth|fds)=\s*([0-9]+),([0-9]+)", make_flags) if make_flags else None
    172         if make_jobs_fds :
    173                 tokens = os.read(int(make_jobs_fds.group(2)), 1024)
    174                 options.jobs = len(tokens)
    175                 os.write(int(make_jobs_fds.group(3)), tokens)
    176         else :
    177                 options.jobs = multiprocessing.cpu_count()
    178 
    179         # make sure we have a valid number of jobs that corresponds to user input
    180         if options.jobs <= 0 :
    181                 print('ERROR: Invalid number of jobs', file=sys.stderr)
    182                 sys.exit(1)
    183 
    184         return min( options.jobs, len(tests) ), True if make_flags else False
    185 
    186 # setup a proper processor pool with correct signal handling
    187 def setupPool(jobs):
    188         original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
    189         pool = multiprocessing.Pool(jobs)
    190         signal.signal(signal.SIGINT, original_sigint_handler)
    191 
    192         return pool
    193 
    194 # handle signals in scope
    195 class SignalHandling():
    196         def __enter__(self):
    197                 # enable signal handling
    198                 signal.signal(signal.SIGINT, signal.SIG_DFL)
    199 
    200         def __exit__(self, type, value, traceback):
    201                 # disable signal handling
    202                 signal.signal(signal.SIGINT, signal.SIG_IGN)
    203 
    204 ################################################################################
    205 #               misc
    206 ################################################################################
    207 
    208 # check if arguments is yes or no
    209 def yes_no(string):
    210         if string == "yes" :
    211                 return True
    212         if string == "no" :
    213                 return False
    214         raise argparse.ArgumentTypeError(msg)
    215         return False
    216 
    217 
    218 settings.set_machine_default( getMachineType )
     96        return sh(diff_cmd % (lhs, rhs), dry_run, False)

  • src/tests/test.py

    r86ad276 r5da9d6a  
    22from __future__ import print_function
    33
     4from functools import partial
     5from multiprocessing import Pool
     6from os import listdir, environ
     7from os.path import isfile, join, splitext
    48from pybin.tools import *
    5 from pybin.test_run import *
    6 from pybin import settings
    79
    810import argparse
     11import multiprocessing
     12import os
    913import re
     14import signal
    1015import sys
    1116
    … …  
    1419################################################################################
    1520
    16 def findTests():
    17         expected = []
    18 
    19         def matchTest(path):
    20                 match = re.search("(\.[\w\/\-_]*)\/.expect\/([\w\-_]+)(\.[\w\-_]+)?\.txt", path)
    21                 if match :
    22                         test = Test()
    23                         test.name = match.group(2)
    24                         test.path = match.group(1)
    25                         test.arch = match.group(3)[1:] if match.group(3) else None
    26                         if settings.arch.match(test.arch):
    27                                 expected.append(test)
    28 
    29         pathWalk( matchTest )
    30 
    31         return expected
     21# Test class that defines what a test is
     22class Test:
     23    def __init__(self, name, path):
     24        self.name, self.path = name, path
     25
     26class TestResult:
     27        SUCCESS = 0
     28        FAILURE = 1
     29        TIMEOUT = 124
     30
     31# parses the Makefile to find the machine type (32-bit / 64-bit)
     32def getMachineType():
     33        sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c')
     34        ret, out = sh("make .dummy -s", print2stdout=True)
     35
     36        if ret != 0:
     37                print("Failed to identify architecture:")
     38                print(out)
     39                print("Stopping")
     40                rm( (".dummy.c",".dummy") )
     41                sys.exit(1)
     42
     43        _, out = sh("file .dummy", print2stdout=False)
     44        rm( (".dummy.c",".dummy") )
     45
     46        return re.search("ELF\s([0-9]+)-bit", out).group(1)
     47
     48def listTestsFolder(folder) :
     49        path = ('./.expect/%s/' % folder) if folder else './.expect/'
     50        subpath = "%s/" % folder if folder else ""
     51
     52        # tests directly in the .expect folder will always be processed
     53        return map(lambda fname: Test(fname, subpath + fname),
     54                [splitext(f)[0] for f in listdir( path )
     55                if not f.startswith('.') and f.endswith('.txt')
     56                ])
    3257
    3358# reads the directory ./.expect and indentifies the tests
    34 def listTests( includes, excludes ):
    35         includes = [canonicalPath( i ) for i in includes] if includes else None
    36         excludes = [canonicalPath( i ) for i in excludes] if excludes else None
     59def listTests( concurrent ):
     60        machineType = getMachineType()
    3761
    3862        # tests directly in the .expect folder will always be processed
    39         test_list = findTests()
    40 
    41         # if we have a limited number of includes, filter by them
    42         if includes:
    43                 test_list = [x for x in test_list if
    44                         x.path.startswith( tuple(includes) )
    45                 ]
    46 
    47         # # if we have a folders to excludes, filter by them
    48         if excludes:
    49                 test_list = [x for x in test_list if not
    50                         x.path.startswith( tuple(excludes) )
    51                 ]
    52 
    53         return test_list
     63        generic_list = listTestsFolder( "" )
     64
     65        # tests in the machineType folder will be ran only for the corresponding compiler
     66        typed_list = listTestsFolder( machineType )
     67
     68        # tests in the concurrent folder will be ran only if concurrency is enabled
     69        concurrent_list = listTestsFolder( "concurrent" ) if concurrent else []
     70
     71        # append both lists to get
     72        return generic_list + typed_list + concurrent_list;
    5473
    5574# from the found tests, filter all the valid tests/desired tests
    … …  
    6180        if options.regenerate_expected :
    6281                for testname in options.tests :
    63                         if Test.valid_name(testname):
    64                                 found = [test for test in allTests if test.target() == testname]
    65                                 tests.append( found[0] if len(found) == 1 else Test.from_target(testname) )
     82                        if testname.endswith( (".c", ".cc", ".cpp") ):
     83                                print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr)
    6684                        else :
    67                                 print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr)
     85                                found = [test for test in allTests if test.name == testname]
     86                                tests.append( found[0] if len(found) == 1 else Test(testname, testname) )
    6887
    6988        else :
    7089                # otherwise we only need to validate that all tests are present in the complete list
    7190                for testname in options.tests:
    72                         test = [t for t in allTests if pathCmp( t.target(), testname )]
    73 
    74                         if test :
     91                        test = [t for t in allTests if t.name == testname]
     92
     93                        if len(test) != 0 :
    7594                                tests.append( test[0] )
    7695                        else :
    … …  
    7897
    7998        # make sure we have at least some test to run
    80         if not tests :
     99        if len(tests) == 0 :
    81100                print('ERROR: No valid test to run', file=sys.stderr)
    82101                sys.exit(1)
    … …  
    89108        parser = argparse.ArgumentParser(description='Script which runs cforall tests')
    90109        parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no')
    91         parser.add_argument('--arch', help='Test for specific architecture', type=str, default='')
     110        parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes')
    92111        parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
    93112        parser.add_argument('--list', help='List all test available', action='store_true')
    … …  
    96115        parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8')
    97116        parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
    98         parser.add_argument('-I','--include', help='Directory of test to include, can be used multiple time, All  if omitted', action='append')
    99         parser.add_argument('-E','--exclude', help='Directory of test to exclude, can be used multiple time, None if omitted', action='append')
    100117        parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
    101118
    … …  
    106123        all_tests  = options.all
    107124        some_tests = len(options.tests) > 0
    108         some_dirs  = len(options.include) > 0 if options.include else 0
    109125
    110126        # check that exactly one of the booleans is set to true
    111         if not sum( (listing, all_tests, some_tests, some_dirs) ) > 0 :
    112                 print('ERROR: must have option \'--all\', \'--list\', \'--include\', \'-I\' or non-empty test list', file=sys.stderr)
     127        if not sum( (listing, all_tests, some_tests) ) == 1 :
     128                print('ERROR: must have option \'--all\', \'--list\' or non-empty test list', file=sys.stderr)
    113129                parser.print_help()
    114130                sys.exit(1)
    … …  
    116132        return options
    117133
     134def jobCount( options ):
     135        # check if the user already passed in a number of jobs for multi-threading
     136        make_flags = environ.get('MAKEFLAGS')
     137        make_jobs_fds = re.search("--jobserver-(auth|fds)=\s*([0-9]+),([0-9]+)", make_flags) if make_flags else None
     138        if make_jobs_fds :
     139                tokens = os.read(int(make_jobs_fds.group(2)), 1024)
     140                options.jobs = len(tokens)
     141                os.write(int(make_jobs_fds.group(3)), tokens)
     142        else :
     143                options.jobs = multiprocessing.cpu_count()
     144
     145        # make sure we have a valid number of jobs that corresponds to user input
     146        if options.jobs <= 0 :
     147                print('ERROR: Invalid number of jobs', file=sys.stderr)
     148                sys.exit(1)
     149
     150        return min( options.jobs, len(tests) ), True if make_flags else False
     151
    118152################################################################################
    119153#               running test functions
    120154################################################################################
    121155# logic to run a single test and return the result (No handling of printing or other test framework logic)
    122 def run_single_test(test):
     156def run_single_test(test, generate, dry_run, debug):
    123157
    124158        # find the output file based on the test name and options flag
    125         out_file = test.target_output()
    126         err_file = test.error_log()
    127         cmp_file = test.expect()
    128         in_file  = test.input()
    129 
    130         # prepare the proper directories
    131         test.prepare()
     159        out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path)
     160        err_file = ".err/%s.log" % test.name
    132161
    133162        # remove any outputs from the previous tests to prevent side effects
    134         rm( (out_file, err_file, test.target()) )
     163        rm( (out_file, err_file, test.name), dry_run )
     164
     165        options = "-debug" if debug else "-nodebug"
    135166
    136167        # build, skipping to next test on error
    137         make_ret, _ = make( test.target(),
    138                 redirects  = "2> %s 1> /dev/null" % out_file,
    139                 error_file = err_file
    140         )
     168        make_ret, _ = sh("""%s test=yes DEBUG_FLAGS="%s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)
     169
     170        retcode = 0
     171        error = None
    141172
    142173        # if the make command succeds continue otherwise skip to diff
    143         if make_ret == 0 or settings.dry_run:
    144                 if settings.dry_run or fileIsExecutable(test.target()) :
     174        if make_ret == 0 :
     175                # fetch optional input
     176                stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.name) else ""
     177
     178                if fileIsExecutable(test.name) :
    145179                        # run test
    146                         retcode, _ = sh("timeout 60 %s > %s 2>&1" % (test.target(), out_file), input = in_file)
     180                        retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run)
    147181                else :
    148182                        # simply cat the result into the output
    149                         retcode, _ = sh("cat %s > %s" % (test.target(), out_file))
    150         else:
    151                 retcode, _ = sh("mv %s %s" % (err_file, out_file))
    152 
     183                        sh("cat %s > %s" % (test.name, out_file), dry_run)
     184
     185        else :
     186                # command failed save the log to less temporary file
     187                sh("mv %s %s" % (err_file, out_file), dry_run)
    153188
    154189        if retcode == 0:
    155                 if settings.generating :
     190                if generate :
    156191                        # if we are ounly generating the output we still need to check that the test actually exists
    157                         if not settings.dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.target()) :
     192                        if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.name) :
    158193                                retcode = 1;
    159                                 error = "\t\tNo make target for test %s!" % test.target()
     194                                error = "\t\tNo make target for test %s!" % test.name
    160195                                sh("rm %s" % out_file, False)
    161                         else:
    162                                 error = None
    163196                else :
    164197                        # fetch return code and error from the diff command
    165                         retcode, error = diff(cmp_file, out_file)
     198                        retcode, error = diff(".expect/%s.txt" % test.path, ".out/%s.log" % test.name, dry_run)
    166199
    167200        else:
    … …  
    171204
    172205        # clean the executable
    173         sh("rm -f %s > /dev/null 2>&1" % test.target())
     206        sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
    174207
    175208        return retcode, error
    176209
    177210# run a single test and handle the errors, outputs, printing, exception handling, etc.
    178 def run_test_worker(t) :
    179 
    180         with SignalHandling():
    181                 # print formated name
    182                 name_txt = "%20s  " % t.name
    183 
    184                 retcode, error = run_single_test(t)
    185 
    186                 # update output based on current action
    187                 result_txt = TestResult.toString( retcode )
    188 
    189                 #print result with error if needed
    190                 text = name_txt + result_txt
    191                 out = sys.stdout
    192                 if error :
    193                         text = text + "\n" + error
    194                         out = sys.stderr
    195 
    196                 print(text, file = out)
    197                 sys.stdout.flush()
    198                 sys.stderr.flush()
     211def run_test_worker(t, generate, dry_run, debug) :
     212
     213        signal.signal(signal.SIGINT, signal.SIG_DFL)
     214        # print formated name
     215        name_txt = "%20s  " % t.name
     216
     217        retcode, error = run_single_test(t, generate, dry_run, debug)
     218
     219        # update output based on current action
     220        if generate :
     221                if   retcode == TestResult.SUCCESS:     result_txt = "Done"
     222                elif retcode == TestResult.TIMEOUT:     result_txt = "TIMEOUT"
     223                else :                                          result_txt = "ERROR code %d" % retcode
     224        else :
     225                if   retcode == TestResult.SUCCESS:     result_txt = "PASSED"
     226                elif retcode == TestResult.TIMEOUT:     result_txt = "TIMEOUT"
     227                else :                                          result_txt = "FAILED with code %d" % retcode
     228
     229        #print result with error if needed
     230        text = name_txt + result_txt
     231        out = sys.stdout
     232        if error :
     233                text = text + "\n" + error
     234                out = sys.stderr
     235
     236        print(text, file = out)
     237        sys.stdout.flush()
     238        sys.stderr.flush()
     239        signal.signal(signal.SIGINT, signal.SIG_IGN)
    199240
    200241        return retcode != TestResult.SUCCESS
    201242
    202243# run the given list of tests with the given parameters
    203 def run_tests(tests, jobs) :
     244def run_tests(tests, generate, dry_run, jobs, debug) :
    204245        # clean the sandbox from previous commands
    205         make('clean', redirects = '> /dev/null 2>&1')
     246        sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
     247
     248        # make sure the required folder are present
     249        sh('mkdir -p .out .expect .err', dry_run)
     250
     251        if generate :
     252                print( "Regenerate tests for: " )
    206253
    207254        # create the executor for our jobs and handle the signal properly
    208         pool = setupPool(jobs)
     255        original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
     256        pool = Pool(jobs)
     257        signal.signal(signal.SIGINT, original_sigint_handler)
    209258
    210259        # for each test to run
    211260        try :
    212                 results = pool.map_async(
    213                         run_test_worker,
    214                         tests,
    215                         chunksize = 1
    216                 ).get(7200)
     261                results = pool.map_async(partial(run_test_worker, generate=generate, dry_run=dry_run, debug=debug), tests, chunksize = 1 ).get(7200)
    217262        except KeyboardInterrupt:
    218263                pool.terminate()
    … …  
    221266
    222267        # clean the workspace
    223         make('clean', redirects = '> /dev/null 2>&1')
     268        sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
    224269
    225270        for failed in results:
    … …  
    240285        options = getOptions()
    241286
    242         # init global settings
    243         settings.init( options )
    244 
    245287        # fetch the liest of all valid tests
    246         allTests = listTests( options.include, options.exclude )
     288        allTests = listTests( options.concurrent )
    247289
    248290        # if user wants all tests than no other treatement of the test list is required
    249         if options.all or options.list or options.list_comp or options.include :
     291        if options.all or options.list or options.list_comp :
    250292                tests = allTests
    251293
    252         #otherwise we need to validate that the test list that was entered is valid
    253         else :
     294        else :
     295                #otherwise we need to validate that the test list that was entered is valid
    254296                tests = validTests( options )
    255297
    256298        # sort the test alphabetically for convenience
    257         tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
     299        tests.sort(key=lambda t: t.name)
    258300
    259301        # users may want to simply list the tests
    260302        if options.list_comp :
    261                 print("-h --help --debug --dry-run --list --arch --all --regenerate-expected -j --jobs ", end='')
    262                 print(" ".join(map(lambda t: "%s" % (t.target()), tests)))
     303                print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')
     304                print(" ".join(map(lambda t: "%s" % (t.name), tests)))
    263305
    264306        elif options.list :
    265                 print("Listing for %s:%s"% (settings.arch.string, settings.debug.string))
    266                 print("\n".join(map(lambda t: "%s" % (t.toString()), tests)))
    267 
    268         else :
    269                 options.jobs, forceJobs = jobCount( options, tests )
    270                 settings.updateMakeCmd(forceJobs, options.jobs)
    271 
    272                 print('%s (%s:%s) on %i cores' % (
    273                         'Regenerate tests' if settings.generating else 'Running',
    274                         settings.arch.string,
    275                         settings.debug.string,
    276                         options.jobs
    277                 ))
     307                print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests)))
     308
     309        else :
     310                options.jobs, forceJobs = jobCount( options )
     311
     312                print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))
     313                make_cmd = "make" if forceJobs else ("make -j%i" % options.jobs)
    278314
    279315                # otherwise run all tests and make sure to return the correct error code
    280                 sys.exit( run_tests(tests, options.jobs) )
     316                sys.exit( run_tests(tests, options.regenerate_expected, options.dry_run, options.jobs, options.debug) )
Note: See TracChangeset for help on using the changeset viewer.