Changeset 0ad0c55


Ignore:
Timestamp:
Dec 4, 2017, 1:19:42 PM (7 years ago)
Author:
Thierry Delisle <tdelisle@…>
Branches:
ADT, aaron-thesis, arm-eh, ast-experimental, cleanup-dtors, deferred_resn, demangler, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, new-env, no_list, persistent-indexer, pthread-emulation, qualifiedEnum, resolv-new, with_gc
Children:
bacc36c
Parents:
b0e5593
Message:

Updated tests script to handle folders

Location:
src/tests
Files:
1 added
4 edited
1 moved

Legend:

Unmodified
Added
Removed
  • src/tests/Makefile.am

    rb0e5593 r0ad0c55  
    2020
    2121if BUILD_CONCURRENCY
    22 concurrent = yes
    23 quick_test += coroutine thread monitor
    24 concurrent_test =               \
    25         coroutine               \
    26         fmtLines                \
    27         pingpong                \
    28         prodcons                \
    29         thread                  \
    30         matrixSum               \
    31         monitor                 \
    32         multi-monitor           \
    33         boundedBuffer           \
    34         preempt                 \
    35         sched-int-block         \
    36         sched-int-disjoint      \
    37         sched-int-wait          \
    38         sched-ext-barge         \
    39         sched-ext-dtor          \
    40         sched-ext-else          \
    41         sched-ext-parse         \
    42         sched-ext-recurse       \
    43         sched-ext-statment      \
    44         sched-ext-when
     22concurrent=
    4523else
    46 concurrent=no
    47 concurrent_test=
     24concurrent='-Econcurrent'
    4825endif
    4926
     
    6239endif
    6340
    64 TEST_FLAGS = $(if $(test), 2> .err/${@}.log, )
     41TEST_FLAGS = $(if $(test), 2> $(test), )
    6542AM_CFLAGS = ${TEST_FLAGS} ${BUILD_FLAGS}
    6643CC = @CFA_BINDIR@/@CFA_NAME@
     
    7047
    7148fstream_test_SOURCES = fstream_test.c
    72 fstream_test_CFLAGS = $(if $(test), 2>> .err/fstream_test.log, ) ${BUILD_FLAGS}
     49fstream_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
    7350
    7451vector_test_SOURCES = vector/vector_int.c vector/array.c vector/vector_test.c
    75 vector_test_CFLAGS = $(if $(test), 2>> .err/vector_test.log, ) ${BUILD_FLAGS}
     52vector_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
    7653
    7754avl_test_SOURCES = avltree/avl_test.c avltree/avl0.c avltree/avl1.c avltree/avl2.c avltree/avl3.c avltree/avl4.c avltree/avl-private.c
    78 avl_test_CFLAGS = $(if $(test), 2>> .err/avl_test.log, ) ${BUILD_FLAGS}
     55avl_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
    7956
    8057all-local :
    81         @+python test.py --debug=${debug} --concurrent=${concurrent} ${quick_test}
     58        @+python test.py --debug=${debug} ${concurrent} ${quick_test}
    8259
    8360all-tests :
    84         @+python test.py --all --debug=${debug} --concurrent=${concurrent}              # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
     61        @+python test.py --all --debug=${debug} ${concurrent}           # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
    8562
    8663clean-local :
     
    8865
    8966list :
    90         @+python test.py --list --concurrent=${concurrent}
     67        @+python test.py --list ${concurrent}
    9168
    9269concurrency :
    93         @+python test.py --debug=${debug} --concurrent=${concurrent} ${concurrent_test}
     70        @+python test.py --debug=${debug} ${concurrent} ${concurrent_test}
    9471
    9572.dummy : .dummy.c @CFA_BINDIR@/@CFA_NAME@
  • src/tests/Makefile.in

    rb0e5593 r0ad0c55  
    9191build_triplet = @build@
    9292host_triplet = @host@
    93 @BUILD_CONCURRENCY_TRUE@am__append_1 = coroutine thread monitor
    94 @BUILD_DEBUG_FALSE@am__append_2 = -nodebug
    95 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_FALSE@am__append_3 = -debug
    96 @BUILD_DEBUG_TRUE@@BUILD_RELEASE_TRUE@am__append_4 = ${DEBUG_FLAGS}
     93@BUILD_DEBUG_FALSE@am__append_1 = -nodebug
     94@BUILD_DEBUG_TRUE@@BUILD_RELEASE_FALSE@am__append_2 = -debug
     95@BUILD_DEBUG_TRUE@@BUILD_RELEASE_TRUE@am__append_3 = ${DEBUG_FLAGS}
    9796EXTRA_PROGRAMS = fstream_test$(EXEEXT) vector_test$(EXEEXT) \
    9897        avl_test$(EXEEXT)
     
    314313top_srcdir = @top_srcdir@
    315314debug = yes
    316 quick_test = vector_test avl_test operators numericConstants \
    317         expression enum array typeof cast dtor-early-exit init_once \
    318         attributes $(am__append_1)
    319 @BUILD_CONCURRENCY_FALSE@concurrent = no
    320 @BUILD_CONCURRENCY_TRUE@concurrent = yes
    321 @BUILD_CONCURRENCY_FALSE@concurrent_test =
    322 @BUILD_CONCURRENCY_TRUE@concurrent_test = \
    323 @BUILD_CONCURRENCY_TRUE@        coroutine               \
    324 @BUILD_CONCURRENCY_TRUE@        fmtLines                \
    325 @BUILD_CONCURRENCY_TRUE@        pingpong                \
    326 @BUILD_CONCURRENCY_TRUE@        prodcons                \
    327 @BUILD_CONCURRENCY_TRUE@        thread                  \
    328 @BUILD_CONCURRENCY_TRUE@        matrixSum               \
    329 @BUILD_CONCURRENCY_TRUE@        monitor                 \
    330 @BUILD_CONCURRENCY_TRUE@        multi-monitor           \
    331 @BUILD_CONCURRENCY_TRUE@        boundedBuffer           \
    332 @BUILD_CONCURRENCY_TRUE@        preempt                 \
    333 @BUILD_CONCURRENCY_TRUE@        sched-int-block         \
    334 @BUILD_CONCURRENCY_TRUE@        sched-int-disjoint      \
    335 @BUILD_CONCURRENCY_TRUE@        sched-int-wait          \
    336 @BUILD_CONCURRENCY_TRUE@        sched-ext-barge         \
    337 @BUILD_CONCURRENCY_TRUE@        sched-ext-dtor          \
    338 @BUILD_CONCURRENCY_TRUE@        sched-ext-else          \
    339 @BUILD_CONCURRENCY_TRUE@        sched-ext-parse         \
    340 @BUILD_CONCURRENCY_TRUE@        sched-ext-recurse       \
    341 @BUILD_CONCURRENCY_TRUE@        sched-ext-statment      \
    342 @BUILD_CONCURRENCY_TRUE@        sched-ext-when
    343 
     315quick_test = vector_test avl_test operators numericConstants expression enum array typeof cast dtor-early-exit init_once attributes
     316@BUILD_CONCURRENCY_FALSE@concurrent = '-Econcurrent'
     317@BUILD_CONCURRENCY_TRUE@concurrent =
    344318
    345319# applies to both programs
    346320DEBUG_FLAGS =
    347321BUILD_FLAGS = -g -Wall -Wno-unused-function -quiet @CFA_FLAGS@ \
    348         $(am__append_2) $(am__append_3) $(am__append_4)
    349 TEST_FLAGS = $(if $(test), 2> .err/${@}.log, )
     322        $(am__append_1) $(am__append_2) $(am__append_3)
     323TEST_FLAGS = $(if $(test), 2> $(test), )
    350324AM_CFLAGS = ${TEST_FLAGS} ${BUILD_FLAGS}
    351325fstream_test_SOURCES = fstream_test.c
    352 fstream_test_CFLAGS = $(if $(test), 2>> .err/fstream_test.log, ) ${BUILD_FLAGS}
     326fstream_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
    353327vector_test_SOURCES = vector/vector_int.c vector/array.c vector/vector_test.c
    354 vector_test_CFLAGS = $(if $(test), 2>> .err/vector_test.log, ) ${BUILD_FLAGS}
     328vector_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
    355329avl_test_SOURCES = avltree/avl_test.c avltree/avl0.c avltree/avl1.c avltree/avl2.c avltree/avl3.c avltree/avl4.c avltree/avl-private.c
    356 avl_test_CFLAGS = $(if $(test), 2>> .err/avl_test.log, ) ${BUILD_FLAGS}
     330avl_test_CFLAGS = $(if $(test), 2>> $(test), ) ${BUILD_FLAGS}
    357331all: all-am
    358332
     
    833807
    834808all-local :
    835         @+python test.py --debug=${debug} --concurrent=${concurrent} ${quick_test}
     809        @+python test.py --debug=${debug} ${concurrent} ${quick_test}
    836810
    837811all-tests :
    838         @+python test.py --all --debug=${debug} --concurrent=${concurrent}              # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
     812        @+python test.py --all --debug=${debug} ${concurrent}           # '@' => do not echo command (SILENT), '+' => allows recursive make from within python program
    839813
    840814clean-local :
     
    842816
    843817list :
    844         @+python test.py --list --concurrent=${concurrent}
     818        @+python test.py --list ${concurrent}
    845819
    846820concurrency :
    847         @+python test.py --debug=${debug} --concurrent=${concurrent} ${concurrent_test}
     821        @+python test.py --debug=${debug} ${concurrent} ${concurrent_test}
    848822
    849823.dummy : .dummy.c @CFA_BINDIR@/@CFA_NAME@
  • src/tests/pybin/tools.py

    rb0e5593 r0ad0c55  
    4646        os.rename(out_fname, fname)
    4747
    48 # helper function to check if a files contains only a spacific string
     48# helper function to check if a files contains only a specific string
    4949def fileContainsOnly(file, text) :
    5050        with open(file) as f:
     
    9595        # fetch return code and error from the diff command
    9696        return sh(diff_cmd % (lhs, rhs), dry_run, False)
     97
     98# parses the Makefile to find the machine type (32-bit / 64-bit)
     99def getMachineType():
     100        return 'x64'
     101        sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c')
     102        ret, out = sh("make .dummy -s", print2stdout=True)
     103
     104        if ret != 0:
     105                print("Failed to identify architecture:")
     106                print(out)
     107                print("Stopping")
     108                rm( (".dummy.c",".dummy") )
     109                sys.exit(1)
     110
     111        _, out = sh("file .dummy", print2stdout=False)
     112        rm( (".dummy.c",".dummy") )
     113
     114        return out
     115        return re.search("ELF\s([0-9]+)-bit", out).group(1)
  • src/tests/test.py

    rb0e5593 r0ad0c55  
    77from os.path import isfile, join, splitext
    88from pybin.tools import *
     9from pybin.test_run import *
    910
    1011import argparse
     
    1920################################################################################
    2021
    21 # Test class that defines what a test is
    22 class Test:
    23     def __init__(self, name, path):
    24         self.name, self.path = name, path
    25 
    26 class TestResult:
    27         SUCCESS = 0
    28         FAILURE = 1
    29         TIMEOUT = 124
    30 
    31 # parses the Makefile to find the machine type (32-bit / 64-bit)
    32 def getMachineType():
    33         sh('echo "void ?{}(int&a,int b){}int main(){return 0;}" > .dummy.c')
    34         ret, out = sh("make .dummy -s", print2stdout=True)
    35 
    36         if ret != 0:
    37                 print("Failed to identify architecture:")
    38                 print(out)
    39                 print("Stopping")
    40                 rm( (".dummy.c",".dummy") )
    41                 sys.exit(1)
    42 
    43         _, out = sh("file .dummy", print2stdout=False)
    44         rm( (".dummy.c",".dummy") )
    45 
    46         return re.search("ELF\s([0-9]+)-bit", out).group(1)
    47 
    48 def listTestsFolder(folder) :
    49         path = ('./.expect/%s/' % folder) if folder else './.expect/'
    50         subpath = "%s/" % folder if folder else ""
     22def list_expected():
     23        expected = []
     24
     25        def step(_, dirname, names):
     26                for name in names:
     27                        path = os.path.join(dirname, name)
     28
     29                        match = re.search("(\.[\w\/\-_]*)\/.expect\/([\w\-_]+)(\.[\w\-_]+)?\.txt", path)
     30                        if match :
     31                                test = Test()
     32                                test.name = match.group(2)
     33                                test.path = match.group(1)
     34                                test.arch = match.group(3)[1:] if match.group(3) else None
     35                                expected.append(test)
     36
     37        # Start the walk
     38        os.path.walk('.', step, '')
     39
     40        return expected
     41
     42# reads the directory ./.expect and indentifies the tests
     43def listTests( includes, excludes ):
     44        includes = [os.path.normpath( os.path.join('.',i) ) for i in includes] if includes else None
     45        excludes = [os.path.normpath( os.path.join('.',i) ) for i in excludes] if excludes else None
    5146
    5247        # tests directly in the .expect folder will always be processed
    53         return map(lambda fname: Test(fname, subpath + fname),
    54                 [splitext(f)[0] for f in listdir( path )
    55                 if not f.startswith('.') and f.endswith('.txt')
    56                 ])
    57 
    58 # reads the directory ./.expect and indentifies the tests
    59 def listTests( concurrent ):
    60         machineType = getMachineType()
    61 
    62         # tests directly in the .expect folder will always be processed
    63         generic_list = listTestsFolder( "" )
    64 
    65         # tests in the machineType folder will be ran only for the corresponding compiler
    66         typed_list = listTestsFolder( machineType )
    67 
    68         # tests in the concurrent folder will be ran only if concurrency is enabled
    69         concurrent_list = listTestsFolder( "concurrent" ) if concurrent else []
    70 
    71         # append both lists to get
    72         return generic_list + typed_list + concurrent_list;
     48        test_list = list_expected()
     49
     50        # if we have a limited number of includes, filter by them
     51        if includes:
     52                test_list = [x for x in test_list if
     53                        os.path.normpath( x.path ).startswith( tuple(includes) )
     54                ]
     55
     56        # # if we have a folders to excludes, filter by them
     57        if excludes:
     58                test_list = [x for x in test_list if not
     59                        os.path.normpath( x.path ).startswith( tuple(excludes) )
     60                ]
     61
     62        return test_list
    7363
    7464# from the found tests, filter all the valid tests/desired tests
     
    8979                # otherwise we only need to validate that all tests are present in the complete list
    9080                for testname in options.tests:
    91                         test = [t for t in allTests if t.name == testname]
     81                        test = [t for t in allTests if os.path.normpath( t.target() ) == os.path.normpath( testname )]
    9282
    9383                        if len(test) != 0 :
     
    10292
    10393        return tests
     94
     95class TestResult:
     96        SUCCESS = 0
     97        FAILURE = 1
     98        TIMEOUT = 124
    10499
    105100# parses the option
     
    108103        parser = argparse.ArgumentParser(description='Script which runs cforall tests')
    109104        parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no')
    110         parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes')
     105        parser.add_argument('--arch', help='Test for specific architecture', type=str, default=getMachineType())
    111106        parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
    112107        parser.add_argument('--list', help='List all test available', action='store_true')
     
    115110        parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8')
    116111        parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
     112        parser.add_argument('-I','--include', help='Directory of test to include, can be used multiple time, All  if omitted', action='append')
     113        parser.add_argument('-E','--exclude', help='Directory of test to exclude, can be used multiple time, None if omitted', action='append')
    117114        parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
    118115
     
    123120        all_tests  = options.all
    124121        some_tests = len(options.tests) > 0
     122        some_dirs  = len(options.include) > 0 if options.include else 0
    125123
    126124        # check that exactly one of the booleans is set to true
    127         if not sum( (listing, all_tests, some_tests) ) == 1 :
    128                 print('ERROR: must have option \'--all\', \'--list\' or non-empty test list', file=sys.stderr)
     125        if not sum( (listing, all_tests, some_tests, some_dirs) ) > 0 :
     126                print('ERROR: must have option \'--all\', \'--list\', \'--include\', \'-I\' or non-empty test list', file=sys.stderr)
    129127                parser.print_help()
    130128                sys.exit(1)
     
    157155
    158156        # find the output file based on the test name and options flag
    159         out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path)
    160         err_file = ".err/%s.log" % test.name
     157        out_file = test.output_file() if not generate else test.expect_file()
     158        err_file = test.error_file()
     159        cmp_file = test.expect_file()
     160        in_file  = test.input_file()
     161
     162        # prepare the proper directories
     163        test.prepare( dry_run )
    161164
    162165        # remove any outputs from the previous tests to prevent side effects
    163         rm( (out_file, err_file, test.name), dry_run )
     166        rm( (out_file, err_file, test.target()), dry_run )
    164167
    165168        options = "-debug" if debug else "-nodebug"
    166169
     170
    167171        # build, skipping to next test on error
    168         make_ret, _ = sh("""%s test=yes DEBUG_FLAGS="%s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)
     172        make_ret, _ = sh("""%s  DEBUG_FLAGS="%s" %s test="%s" 2> %s 1> /dev/null""" % (make_cmd, options, test.target(), err_file, out_file), dry_run)
    169173
    170174        retcode = 0
     
    172176
    173177        # if the make command succeds continue otherwise skip to diff
    174         if make_ret == 0 :
     178        if make_ret == 0 or dry_run:
    175179                # fetch optional input
    176                 stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.name) else ""
    177 
    178                 if fileIsExecutable(test.name) :
     180                stdinput = "< %s" % in_file if isfile(in_file) else ""
     181
     182                if dry_run or fileIsExecutable(test.target()) :
    179183                        # run test
    180                         retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run)
     184                        retcode, _ = sh("timeout 60 ./%s %s > %s 2>&1" % (test.target(), stdinput, out_file), dry_run)
    181185                else :
    182186                        # simply cat the result into the output
    183                         sh("cat %s > %s" % (test.name, out_file), dry_run)
    184 
    185         else :
    186                 # command failed save the log to less temporary file
     187                        sh("cat %s > %s" % (test.target(), out_file), dry_run)
     188        else:
    187189                sh("mv %s %s" % (err_file, out_file), dry_run)
     190
    188191
    189192        if retcode == 0:
    190193                if generate :
    191194                        # if we are ounly generating the output we still need to check that the test actually exists
    192                         if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.name) :
     195                        if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.target()) :
    193196                                retcode = 1;
    194                                 error = "\t\tNo make target for test %s!" % test.name
     197                                error = "\t\tNo make target for test %s!" % test.target()
    195198                                sh("rm %s" % out_file, False)
    196199                else :
    197200                        # fetch return code and error from the diff command
    198                         retcode, error = diff(".expect/%s.txt" % test.path, ".out/%s.log" % test.name, dry_run)
     201                        retcode, error = diff(cmp_file, out_file, dry_run)
    199202
    200203        else:
     
    204207
    205208        # clean the executable
    206         sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
     209        sh("rm -f %s > /dev/null 2>&1" % test.target(), dry_run)
    207210
    208211        return retcode, error
     
    245248        # clean the sandbox from previous commands
    246249        sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
    247 
    248         # make sure the required folder are present
    249         sh('mkdir -p .out .expect .err', dry_run)
    250250
    251251        if generate :
     
    286286
    287287        # fetch the liest of all valid tests
    288         allTests = listTests( options.concurrent )
     288        allTests = listTests( options.include, options.exclude )
    289289
    290290        # if user wants all tests than no other treatement of the test list is required
    291         if options.all or options.list or options.list_comp :
     291        if options.all or options.list or options.list_comp or options.include :
    292292                tests = allTests
    293293
     
    297297
    298298        # sort the test alphabetically for convenience
    299         tests.sort(key=lambda t: t.name)
     299        tests.sort(key=lambda t: os.path.join(t.path, t.name))
    300300
    301301        # users may want to simply list the tests
    302302        if options.list_comp :
    303                 print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')
    304                 print(" ".join(map(lambda t: "%s" % (t.name), tests)))
     303                print("-h --help --debug --dry-run --list --all --regenerate-expected -j --jobs ", end='')
     304                print(" ".join(map(lambda t: "%s" % (t.target()), tests)))
    305305
    306306        elif options.list :
    307                 print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests)))
     307                print("Listing for %s:%s"% (options.arch, "debug" if options.debug else "no debug"))
     308                print("\n".join(map(lambda t: "%s" % (t.toString()), tests)))
    308309
    309310        else :
    310311                options.jobs, forceJobs = jobCount( options )
    311312
    312                 print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))
     313                print('Running (%s:%s) on %i cores' % (options.arch, "debug" if options.debug else "no debug", options.jobs))
    313314                make_cmd = "make" if forceJobs else ("make -j%i" % options.jobs)
    314315
Note: See TracChangeset for help on using the changeset viewer.