Index: Jenkins/Distribute =================================================================== --- Jenkins/Distribute (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ Jenkins/Distribute (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -15,4 +15,5 @@ SrcDir = pwd tmp: false Settings = null + Version = '' // Local variables @@ -22,54 +23,34 @@ currentBuild.result = "SUCCESS" - try { - //Wrap build to add timestamp to command line - wrap([$class: 'TimestamperBuildWrapper']) { + //Wrap build to add timestamp to command line + wrap([$class: 'TimestamperBuildWrapper']) { - final commit, build - (commit, build) = prepare_build() + final commit, build + (commit, build) = prepare_build() - node('x64') { - BuildDir = pwd tmp: true - SrcDir = pwd tmp: false - - Tools.Clean() - - Tools.Checkout( commit ) - - final version = GetVersion( build ) - } - - // Update the build directories when exiting the node + node('x64') { BuildDir = pwd tmp: true SrcDir = pwd tmp: false + + Tools.Clean() + + Tools.Checkout( commit ) + + Version = GetVersion( build ) + + Configure() + + Package() + + Test() + + Archive() } + + // Update the build directories when exiting the node + BuildDir = pwd tmp: true + SrcDir = pwd tmp: false } - //If an exception is caught we need to change the status and remember to - //attach the build log to the email - // catch (Exception caughtError) { - // //rethrow error later - // err = caughtError - - // echo err.toString() - - // //An error has occured, the build log is relevent - // log_needed = true - - // //Store the result of the build log - // currentBuild.result = "${StageName} FAILURE".trim() - // } - - finally { - // //Send email with final results if this is not a full build - // email(log_needed) - - // echo 'Distribution Completed' - - // /* Must re-throw exception to propagate error */ - // if (err) { - // throw err - // } - } } @@ -90,4 +71,49 @@ return version +} + +def Configure() { + Tools.BuildStage('Configure', true) { + // Configure must be run inside the tree + dir (SrcDir) { + // Generate the necessary build files + sh './autogen.sh' + } + + // Build outside of the src tree to ease cleaning + dir (BuildDir) { + // Configure the compilation (Output is not relevant) + // Use the current directory as the installation target so nothing escapes the sandbox + // Also specify the compiler by hand + sh "${SrcDir}/configure CXX=g++-9 CC=gcc-9 AR=gcc-ar RANLIB=gcc-ranlib --quiet" + + // Configure libcfa + sh 'make -j 8 --no-print-directory configure-libcfa' + } + } +} + +def Package() { + Tools.BuildStage('Package', true) { + dir (BuildDir) { + sh "make VERSION=${Version} dist" + } + } +} + +def Test() { + Tools.BuildStage('Test', true) { + dir (BuildDir) { + sh "make VERSION=${Version} distcheck -j 8" + } + } +} + +def Archive() { + Tools.BuildStage('Archive', true) { + dir (BuildDir) { + archiveArtifacts artifacts: "cfa-cc-*.tar.gz", fingerprint: true + } + } } Index: Jenkins/FullBuild =================================================================== --- Jenkins/FullBuild (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ Jenkins/FullBuild (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -26,6 +26,10 @@ gcc_5_x64_old: { trigger_build( 'gcc-5', 'x64', false ) }, clang_x64_old: { trigger_build( 'clang', 'x64', false ) }, - // clang_x64_new: { trigger_build( 'clang', 'x64', true ) }, + clang_x64_new: { trigger_build( 'clang', 'x64', true ) }, ) + } + + stage('Package') { + build job: 'Cforall_Distribute_Ref', parameters: [string(name: 'GitRef', value: gitRefNewValue), string(name: 'Build', value: currentBuild.number)] } } Index: Jenkinsfile =================================================================== --- Jenkinsfile (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ Jenkinsfile (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -93,5 +93,5 @@ // Build outside of the src tree to ease cleaning dir (BuildDir) { - //Configure the conpilation (Output is not relevant) + //Configure the compilation (Output is not relevant) //Use the current directory as the installation target so nothing escapes the sandbox //Also specify the compiler by hand Index: Makefile.am =================================================================== --- Makefile.am (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ Makefile.am (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -21,5 +21,5 @@ SUBDIRS = driver src . @LIBCFA_TARGET_DIRS@ -DIST_SUBDIRS = driver src . libcfa benchmark longrun_tests tests tools tools/prettyprinter +DIST_SUBDIRS = driver src . libcfa tests tools tools/prettyprinter @LIBCFA_TARGET_MAKEFILES@ : Makefile $(srcdir)/libcfa/configure Index: configure.ac =================================================================== --- configure.ac (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ configure.ac (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -3,5 +3,5 @@ AC_PREREQ([2.68]) -AC_INIT([cfa-cc],[1.0.0.0],[cforall@plg.uwaterloo.ca]) +AC_INIT([cfa-cc],[1.0.0],[cforall@plg.uwaterloo.ca]) AC_CONFIG_AUX_DIR([automake]) AC_CONFIG_MACRO_DIRS([automake]) @@ -28,5 +28,5 @@ # New AST toggling support AH_TEMPLATE([CFA_USE_NEW_AST],[Sets whether or not to use the new-ast, this is adefault value and can be overrided by --old-ast and --new-ast]) -DEFAULT_NEW_AST="False" +DEFAULT_NEW_AST="True" AC_ARG_ENABLE(new-ast, [ --enable-new-ast whether or not to use new ast as the default AST algorithm], @@ -35,5 +35,5 @@ no) newast=false; DEFAULT_NEW_AST="False" ;; *) AC_MSG_ERROR([bad value ${enableval} for --enable-new-ast]) ;; - esac],[newast=false]) + esac],[newast=true]) AC_DEFINE_UNQUOTED([CFA_USE_NEW_AST], $newast) AC_SUBST(DEFAULT_NEW_AST) @@ -103,6 +103,6 @@ # Create variables for commonly used targets -TOP_SRCDIR="$(readlink -m $ac_confdir/)/" -TOP_BUILDDIR="$(readlink -m $ac_pwd/)/" +TOP_SRCDIR="$(readlink -e $ac_abs_confdir/)/" +TOP_BUILDDIR="$(readlink -e $ac_pwd/)/" AC_DEFINE_UNQUOTED(TOP_SRCDIR, "$TOP_SRCDIR", [Top src directory]) @@ -139,4 +139,8 @@ \'--enable-gprofiler=*) ;; \'--disable-gprofiler) ;; + + # skip the target hosts + \'--enable-new-ast=*) ;; + \'--disable-new-ast) ;; # skip this, it only causes problems @@ -287,11 +291,17 @@ libcfa/Makefile:libcfa/Makefile.dist.in tests/Makefile - longrun_tests/Makefile - benchmark/Makefile - benchmark/io/http/Makefile tools/Makefile tools/prettyprinter/Makefile ]) +# Some of our makefile don't need to be distributed +AM_CONDITIONAL([CFORALL_DISTRIBUTE], [test -e $TOP_SRCDIR/autogen.sh]) +AM_COND_IF([CFORALL_DISTRIBUTE], + [AC_CONFIG_FILES([ + longrun_tests/Makefile + benchmark/Makefile + benchmark/io/http/Makefile + ])]) + AC_CONFIG_LINKS([tests/test.py:tests/test.py]) Index: libcfa/configure.ac =================================================================== --- libcfa/configure.ac (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ libcfa/configure.ac (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -3,5 +3,5 @@ AC_PREREQ([2.68]) -AC_INIT([cfa-cc],[1.0.0.0],[cforall@plg.uwaterloo.ca]) +AC_INIT([cfa-cc],[1.0.0],[cforall@plg.uwaterloo.ca]) AC_CONFIG_AUX_DIR([automake]) AC_CONFIG_MACRO_DIRS([automake]) Index: libcfa/src/Makefile.am =================================================================== --- libcfa/src/Makefile.am (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ libcfa/src/Makefile.am (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -43,4 +43,5 @@ clock.hfa \ exception.hfa \ + exception.h \ gmp.hfa \ math.hfa \ @@ -77,5 +78,4 @@ bits/debug.cfa \ exception.c \ - exception.h \ interpose.cfa \ lsda.h \ @@ -87,5 +87,7 @@ # not all platforms support concurrency, add option do disable it inst_thread_headers_nosrc = \ + bits/collection.hfa \ bits/random.hfa \ + bits/sequence.hfa \ concurrency/clib/cfathread.h \ concurrency/invoke.h \ Index: libcfa/src/concurrency/preemption.cfa =================================================================== --- libcfa/src/concurrency/preemption.cfa (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ libcfa/src/concurrency/preemption.cfa (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -405,5 +405,5 @@ #define RELOC_SUFFIX "" #endif - #define __cfaasm_label( label ) static struct asm_region label = \ + #define __cfaasm_label( label ) struct asm_region label = \ ({ \ struct asm_region region; \ @@ -424,5 +424,5 @@ #define RELOC_SUFFIX "" #endif - #define __cfaasm_label( label ) static struct asm_region label = \ + #define __cfaasm_label( label ) struct asm_region label = \ ({ \ struct asm_region region; \ @@ -437,5 +437,5 @@ #ifdef __PIC__ // Note that this works only for gcc - #define __cfaasm_label( label ) static struct asm_region label = \ + #define __cfaasm_label( label ) struct asm_region label = \ ({ \ struct asm_region region; \ @@ -452,5 +452,5 @@ #error this is not the right thing to do /* - #define __cfaasm_label( label ) static struct asm_region label = \ + #define __cfaasm_label( label ) struct asm_region label = \ ({ \ struct asm_region region; \ Index: src/AST/module.mk =================================================================== --- src/AST/module.mk (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ src/AST/module.mk (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -59,4 +59,5 @@ AST/SymbolTable.cpp \ AST/SymbolTable.hpp \ + AST/TranslationUnit.hpp \ AST/Type.cpp \ AST/Type.hpp \ Index: src/Common/CodeLocationTools.cpp =================================================================== --- src/Common/CodeLocationTools.cpp (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ src/Common/CodeLocationTools.cpp (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -27,46 +27,4 @@ struct has_code_location : public std::is_base_of {}; -// Fill every location -class FillCore : public ast::WithGuards { - CodeLocation const * parent; - - // This mimics a partially specialized method. - template - struct FillNode; - template - friend struct FillNode; - - template - struct FillNode { - static node_t const * go( FillCore * core, node_t const * node ) { - node_t * newNode = nullptr; - if ( node->location.isUnset() ) { - // Just hoping that top level nodes are always marked. - assert( core->parent ); - newNode = ast::mutate( node ); - newNode->location = *core->parent; - } - core->GuardValue( core->parent ); - core->parent = &node->location; - return (newNode) ? newNode : node; - } - }; - - template - struct FillNode { - static node_t const * go( FillCore *, node_t const * node ) { - return node; - } - }; -public: - FillCore() : parent( nullptr ) {} - - template - node_t const * previsit( node_t const * node ) { - using Filler = FillNode::value>; - return Filler::go( this, node ); - } -}; - template struct __GetCL; @@ -74,5 +32,9 @@ template struct __GetCL { - static CodeLocation const * get( node_t const * node ) { + static inline CodeLocation const * get( node_t const * node ) { + return &node->location; + } + + static inline CodeLocation * get( node_t * node ) { return &node->location; } @@ -81,5 +43,5 @@ template struct __GetCL { - static CodeLocation const * get( node_t const * ) { + static inline CodeLocation * get( node_t const * ) { return nullptr; } @@ -90,4 +52,34 @@ return __GetCL< node_t, has_code_location< node_t >::value >::get( node ); } + +template +CodeLocation * get_code_location( node_t * node ) { + return __GetCL< node_t, has_code_location< node_t >::value >::get( node ); +} + +// Fill every location with a nearby (parent) location. +class FillCore : public ast::WithGuards { + CodeLocation const * parent; +public: + FillCore() : parent( nullptr ) {} + + template + node_t const * previsit( node_t const * node ) { + GuardValue( parent ); + CodeLocation const * location = get_code_location( node ); + if ( location && location->isUnset() ) { + assert( parent ); + node_t * newNode = ast::mutate( node ); + CodeLocation * newLocation = get_code_location( newNode ); + assert( newLocation ); + *newLocation = *parent; + parent = newLocation; + return newNode; + } else if ( location ) { + parent = location; + } + return node; + } +}; // Collect pointers to all the nodes with unset code locations. Index: tests/Makefile.am =================================================================== --- tests/Makefile.am (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ tests/Makefile.am (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -68,4 +68,26 @@ .INTERMEDIATE: .validate .validate.cfa EXTRA_PROGRAMS = avl_test .dummy_hack # build but do not install +EXTRA_DIST = test.py \ + pybin/__init__.py \ + pybin/print-core.gdb \ + pybin/settings.py \ + pybin/test_run.py \ + pybin/tools.py \ + long_tests.hfa \ + .in/io.data \ + avltree/avl.h \ + avltree/avl-private.h \ + concurrent/clib.c \ + exceptions/with-threads.hfa \ + exceptions/except-io.hfa + +dist-hook: + echo "Gathering test files" + for file in `${TEST_PY} --list-dist`; do \ + if test -f ${srcdir}/$${file}; then \ + $(MKDIR_P) $$(dirname ${distdir}/$${file}); \ + cp -df ${srcdir}/$${file} ${distdir}/$${file}; \ + fi; \ + done avl_test_SOURCES = avltree/avl_test.cfa avltree/avl0.cfa avltree/avl1.cfa avltree/avl2.cfa avltree/avl3.cfa avltree/avl4.cfa avltree/avl-private.cfa Index: tests/pybin/tools.py =================================================================== --- tests/pybin/tools.py (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ tests/pybin/tools.py (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -191,4 +191,15 @@ cmd = [s for s in cmd if s] return sh(*cmd, output_file=output_file, error=error) + +def make_recon(target): + cmd = [ + *settings.make, + '-W', + os.path.abspath(os.path.join(settings.BUILDDIR, '../driver/cfa')), + '--recon', + target + ] + cmd = [s for s in cmd if s] + return sh(*cmd, output_file=subprocess.PIPE) def which(program): Index: tests/test.py =================================================================== --- tests/test.py (revision 0f7a0ea0cacf9a2f12e9bb9ef5a12ef3ec94f57a) +++ tests/test.py (revision 48f625297aea3f44ac24a93c92f6eabef922fc90) @@ -143,4 +143,5 @@ parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int) parser.add_argument('--list-comp', help='List all valide arguments', action='store_true') + parser.add_argument('--list-dist', help='List all tests for distribution', action='store_true') parser.add_argument('-I','--include', help='Directory of test to include, can be used multiple time, All if omitted', action='append') parser.add_argument('-E','--exclude', help='Directory of test to exclude, can be used multiple time, None if omitted', action='append') @@ -155,5 +156,5 @@ # script must have at least some tests to run or be listing - listing = options.list or options.list_comp + listing = options.list or options.list_comp or options.list_dist all_tests = options.all some_tests = len(options.tests) > 0 @@ -334,5 +335,7 @@ settings.init( options ) - # users may want to simply list the tests + # -------------------------------------------------- + # list all the test for auto completion programs + # not pretty, single line, with the command line options if options.list_comp : # fetch the liest of all valid tests @@ -340,8 +343,36 @@ # print the possible options - print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs -I --include -E --exclude --continue ", end='') + print("-h --help --debug --dry-run --list --ast=new --ast=old --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs -I --include -E --exclude --continue ", end='') print(" ".join(map(lambda t: "%s" % (t.target()), tests))) - elif options.list : + # done + sys.exit(0) + + # -------------------------------------------------- + # list all the test for auto completion programs + if options.list_dist : + # fetch the liest of all valid tests + tests = list_tests( None, None ) + + for t in tests: + print(os.path.relpath(t.expect(), settings.SRCDIR), end=' ') + print(os.path.relpath(t.input() , settings.SRCDIR), end=' ') + code, out = make_recon(t.target()) + + if code != 0: + print('ERROR: recond failed for test {}'.format(t.target()), file=sys.stderr) + sys.exit(1) + + print(' '.join(re.findall('([^\s]+\.cfa)', out)), end=' ') + + print('') + + # done + sys.exit(0) + + + # -------------------------------------------------- + # list all the tests for users, in a pretty format + if options.list : # fetch the liest of all valid tests tests = list_tests( options.include, options.exclude ) @@ -350,69 +381,71 @@ fancy_print("\n".join(map(lambda t: t.toString(), tests))) + # done + sys.exit(0) + + # fetch the liest of all valid tests + all_tests = list_tests( options.include, options.exclude ) + + # if user wants all tests than no other treatement of the test list is required + if options.all or options.include : + tests = all_tests + + #otherwise we need to validate that the test list that was entered is valid else : - # fetch the liest of all valid tests - all_tests = list_tests( options.include, options.exclude ) - - # if user wants all tests than no other treatement of the test list is required - if options.all or options.include : - tests = all_tests - - #otherwise we need to validate that the test list that was entered is valid - else : - tests = valid_tests( options ) - - # make sure we have at least some test to run - if not tests : - print('ERROR: No valid test to run', file=sys.stderr) - sys.exit(1) - - # prep invariants - settings.prep_output(tests) - failed = 0 - - # check if the expected files aren't empty - if not options.regenerate_expected: - for t in tests: - if is_empty(t.expect()): - print('WARNING: test "{}" has empty .expect file'.format(t.target()), file=sys.stderr) - - # for each build configurations, run the test - with Timed() as total_dur: - for ast, arch, debug, install in itertools.product(settings.all_ast, settings.all_arch, settings.all_debug, settings.all_install): - settings.ast = ast - settings.arch = arch - settings.debug = debug - settings.install = install - - # filter out the tests for a different architecture - # tests are the same across debug/install - local_tests = settings.ast.filter( tests ) - local_tests = settings.arch.filter( local_tests ) - options.jobs, forceJobs = job_count( options, local_tests ) - settings.update_make_cmd(forceJobs, options.jobs) - - # check the build configuration works - settings.validate() - - # print configuration - print('%s %i tests on %i cores (%s:%s - %s)' % ( - 'Regenerating' if settings.generating else 'Running', - len(local_tests), - options.jobs, - settings.ast.string, - settings.arch.string, - settings.debug.string - )) - if not local_tests : - print('WARNING: No tests for this configuration') - continue - - # otherwise run all tests and make sure to return the correct error code - failed = run_tests(local_tests, options.jobs) - if failed: - result = 1 - if not settings.continue_: - break - - print('Tests took %s' % fmtDur( total_dur.duration )) - sys.exit( failed ) + tests = valid_tests( options ) + + # make sure we have at least some test to run + if not tests : + print('ERROR: No valid test to run', file=sys.stderr) + sys.exit(1) + + # prep invariants + settings.prep_output(tests) + failed = 0 + + # check if the expected files aren't empty + if not options.regenerate_expected: + for t in tests: + if is_empty(t.expect()): + print('WARNING: test "{}" has empty .expect file'.format(t.target()), file=sys.stderr) + + # for each build configurations, run the test + with Timed() as total_dur: + for ast, arch, debug, install in itertools.product(settings.all_ast, settings.all_arch, settings.all_debug, settings.all_install): + settings.ast = ast + settings.arch = arch + settings.debug = debug + settings.install = install + + # filter out the tests for a different architecture + # tests are the same across debug/install + local_tests = settings.ast.filter( tests ) + local_tests = settings.arch.filter( local_tests ) + options.jobs, forceJobs = job_count( options, local_tests ) + settings.update_make_cmd(forceJobs, options.jobs) + + # check the build configuration works + settings.validate() + + # print configuration + print('%s %i tests on %i cores (%s:%s - %s)' % ( + 'Regenerating' if settings.generating else 'Running', + len(local_tests), + options.jobs, + settings.ast.string, + settings.arch.string, + settings.debug.string + )) + if not local_tests : + print('WARNING: No tests for this configuration') + continue + + # otherwise run all tests and make sure to return the correct error code + failed = run_tests(local_tests, options.jobs) + if failed: + result = 1 + if not settings.continue_: + break + + print('Tests took %s' % fmtDur( total_dur.duration )) + sys.exit( failed )