#!groovy import groovy.transform.Field // For skipping stages import org.jenkinsci.plugins.pipeline.modeldefinition.Utils //=========================================================================================================== // Main loop of the compilation //=========================================================================================================== node('master') { // Globals BuildDir = pwd tmp: true SrcDir = pwd tmp: false Settings = null StageName = '' // Local variables def err = null def log_needed = false currentBuild.result = "SUCCESS" try { //Wrap build to add timestamp to command line wrap([$class: 'TimestamperBuildWrapper']) { Settings = prepare_build() node(Settings.Architecture.node) { BuildDir = pwd tmp: true SrcDir = pwd tmp: false clean() checkout() build() test() benchmark() build_doc() publish() } // Update the build directories when exiting the node BuildDir = pwd tmp: true SrcDir = pwd tmp: false } } //If an exception is caught we need to change the status and remember to //attach the build log to the email catch (Exception caughtError) { //rethrow error later err = caughtError echo err.toString() //An error has occured, the build log is relevent log_needed = true //Store the result of the build log currentBuild.result = "${StageName} FAILURE".trim() } finally { //Send email with final results if this is not a full build email(log_needed) echo 'Build Completed' /* Must re-throw exception to propagate error */ if (err) { throw err } } } //=========================================================================================================== // Main compilation routines //=========================================================================================================== def clean() { build_stage('Cleanup', true) { // clean the build by wipping the build directory dir(BuildDir) { deleteDir() } } } //Compilation script is done here but environnement set-up and error handling is done in main loop def checkout() { build_stage('Checkout', true) { //checkout the source code and clean the repo final scmVars = checkout scm Settings.GitNewRef = scmVars.GIT_COMMIT Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT echo GitLogMessage() // This is a complete hack but it solves problems with automake thinking it needs to regenerate makefiles // We fudged automake/missing to handle that but automake stills bakes prints inside the makefiles // and these cause more problems. sh 'find . -name Makefile.in -exec touch {} +' } } def build() { debug = true release = Settings.RunAllTests || Settings.RunBenchmark build_stage('Build : configure', true) { // Configure must be run inside the tree dir (SrcDir) { // Generate the necessary build files sh './autogen.sh' } // Build outside of the src tree to ease cleaning dir (BuildDir) { //Configure the conpilation (Output is not relevant) //Use the current directory as the installation target so nothing escapes the sandbox //Also specify the compiler by hand targets="" if( Settings.RunAllTests || Settings.RunBenchmark ) { targets="--with-target-hosts='host:debug,host:nodebug'" } else { targets="--with-target-hosts='host:debug'" } sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} --quiet --prefix=${BuildDir}" // Configure libcfa sh 'make -j 8 --no-print-directory configure-libcfa' } } build_stage('Build : cfa-cpp', true) { // Build outside of the src tree to ease cleaning dir (BuildDir) { // Build driver sh 'make -j 8 --no-print-directory -C driver' // Build translator sh 'make -j 8 --no-print-directory -C src' } } build_stage('Build : libcfa(debug)', debug) { // Build outside of the src tree to ease cleaning dir (BuildDir) { sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug" } } build_stage('Build : libcfa(nodebug)', release) { // Build outside of the src tree to ease cleaning dir (BuildDir) { sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug" } } build_stage('Build : install', true) { // Build outside of the src tree to ease cleaning dir (BuildDir) { sh "make -j 8 --no-print-directory install" } } } def test() { try { build_stage('Test: short', !Settings.RunAllTests) { dir (BuildDir) { //Run the tests from the tests directory sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short" } } build_stage('Test: full', Settings.RunAllTests) { dir (BuildDir) { //Run the tests from the tests directory sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug""" sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug""" } } } catch (Exception err) { echo "Archiving core dumps" dir (BuildDir) { archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true } throw err } } def benchmark() { build_stage('Benchmark', Settings.RunBenchmark) { dir (BuildDir) { //Append bench results sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}" } } } def build_doc() { build_stage('Documentation', Settings.BuildDocumentation) { dir ('doc/user') { make_doc() } dir ('doc/refrat') { make_doc() } } } def publish() { build_stage('Publish', true) { if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' } def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true) def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false) //Then publish the results do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation') do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)') do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching') do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)') do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion') do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)') do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling') do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)') } } //=========================================================================================================== //Routine responsible of sending the email notification once the build is completed //=========================================================================================================== @NonCPS def SplitLines(String text) { def list = [] text.eachLine { list += it } return list } def GitLogMessage() { if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n" def oldRef = Settings.GitOldRef def newRef = Settings.GitNewRef def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim() def revList = SplitLines( revText ) def gitUpdate = "" revList.each { rev -> def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim() gitUpdate = gitUpdate + " via ${rev} (${type})" } def rev = oldRef def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim() gitUpdate = gitUpdate + " from ${rev} (${type})" def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim() def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim() gitDiff = gitDiff.replace('', '') gitDiff = gitDiff.replace('', '') gitDiff = gitDiff.replace('', '') return """
The branch ${env.BRANCH_NAME} has been updated.
${gitUpdate}

Check console output at ${env.BUILD_URL} to view the results.

- Status --------------------------------------------------------------

BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}

- Log -----------------------------------------------------------------

${gitLog}

-----------------------------------------------------------------------

Summary of changes:
${gitDiff}
""" } //Standard build email notification def email(boolean log) { //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line //Configurations for email format echo 'Notifying users of result' def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase() def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}" def email_body = """

This is an automated email from the Jenkins build machine. It was generated because of a git hooks/post-receive script following a ref change which was pushed to the C\u2200 repository.

""" + GitLogMessage() def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca" if( Settings && !Settings.Silent ) { //send email notification emailext body: email_body, subject: email_subject, to: email_to, attachLog: log } else { echo "Would send email to: ${email_to}" echo "With title: ${email_subject}" echo "Content: \n${email_body}" } } //=========================================================================================================== // Helper classes/variables/routines //=========================================================================================================== //Description of a compiler (Must be serializable since pipelines are persistent) class CC_Desc implements Serializable { public String name public String CXX public String CC public String lto CC_Desc(String name, String CXX, String CC, String lto) { this.name = name this.CXX = CXX this.CC = CC this.lto = lto } } //Description of an architecture (Must be serializable since pipelines are persistent) class Arch_Desc implements Serializable { public String name public String flags public String node Arch_Desc(String name, String flags, String node) { this.name = name this.flags = flags this.node = node } } class BuildSettings implements Serializable { public final CC_Desc Compiler public final Arch_Desc Architecture public final Boolean RunAllTests public final Boolean RunBenchmark public final Boolean BuildDocumentation public final Boolean Publish public final Boolean Silent public final Boolean IsSandbox public final String DescLong public final String DescShort public String GitNewRef public String GitOldRef BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) { switch( param.Compiler ) { case 'gcc-9': this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto') break case 'gcc-8': this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto') break case 'gcc-7': this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto') break case 'gcc-6': this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto') break case 'gcc-5': this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto') break case 'gcc-4.9': this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto') break case 'clang': this.Compiler = new CC_Desc('clang', 'clang++-6.0', 'gcc-6', '-flto=thin -flto-jobs=0') break default : error "Unhandled compiler : ${cc}" } switch( param.Architecture ) { case 'x64': this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64') break case 'x86': this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86') break default : error "Unhandled architecture : ${arch}" } this.IsSandbox = (branch == "jenkins-sandbox") this.RunAllTests = param.RunAllTests this.RunBenchmark = param.RunBenchmark this.BuildDocumentation = param.BuildDocumentation this.Publish = param.Publish this.Silent = param.Silent def full = param.RunAllTests ? " (Full)" : "" this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}" this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC }) Architecture : ${ this.Architecture.name } Arc Flags : ${ this.Architecture.flags } Run All Tests : ${ this.RunAllTests.toString() } Run Benchmark : ${ this.RunBenchmark.toString() } Build Documentation : ${ this.BuildDocumentation.toString() } Publish : ${ this.Publish.toString() } Silent : ${ this.Silent.toString() } """ this.GitNewRef = '' this.GitOldRef = '' } } class PlotGroup implements Serializable { public String name public String unit public boolean log PlotGroup(String name, String unit, boolean log) { this.name = name this.unit = unit this.log = log } } def prepare_build() { // prepare the properties properties ([ \ buildDiscarder(logRotator( \ artifactDaysToKeepStr: '', \ artifactNumToKeepStr: '', \ daysToKeepStr: '730', \ numToKeepStr: '1000' \ )), \ [$class: 'ParametersDefinitionProperty', \ parameterDefinitions: [ \ [$class: 'ChoiceParameterDefinition', \ description: 'Which compiler to use', \ name: 'Compiler', \ choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \ defaultValue: 'gcc-8', \ ], \ [$class: 'ChoiceParameterDefinition', \ description: 'The target architecture', \ name: 'Architecture', \ choices: 'x64\nx86', \ defaultValue: 'x64', \ ], \ [$class: 'BooleanParameterDefinition', \ description: 'If false, only the quick test suite is ran', \ name: 'RunAllTests', \ defaultValue: false, \ ], \ [$class: 'BooleanParameterDefinition', \ description: 'If true, jenkins also runs benchmarks', \ name: 'RunBenchmark', \ defaultValue: false, \ ], \ [$class: 'BooleanParameterDefinition', \ description: 'If true, jenkins also builds documentation', \ name: 'BuildDocumentation', \ defaultValue: true, \ ], \ [$class: 'BooleanParameterDefinition', \ description: 'If true, jenkins also publishes results', \ name: 'Publish', \ defaultValue: false, \ ], \ [$class: 'BooleanParameterDefinition', \ description: 'If true, jenkins will not send emails', \ name: 'Silent', \ defaultValue: false, \ ], \ ], ]]) // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer checkout scm final settings = new BuildSettings(params, env.BRANCH_NAME) currentBuild.description = settings.DescShort echo settings.DescLong return settings } def build_stage(String name, boolean run, Closure block ) { StageName = name echo " -------- ${StageName} -------- " if(run) { stage(name, block) } else { stage(name) { Utils.markStageSkippedForConditional(STAGE_NAME) } } } def make_doc() { def err = null try { sh 'make clean > /dev/null' sh 'make > /dev/null 2>&1' } catch (Exception caughtError) { err = caughtError //rethrow error later sh 'cat build/*.log' } finally { if (err) throw err // Must re-throw exception to propagate error } } def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) { if(new_data) { echo "Publishing new data" } def series = new_data ? [[ file: "${file}.csv", exclusionValues: '', displayTableFlag: false, inclusionFlag: 'OFF', url: '' ]] : []; echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}" dir("${BuildDir}/benchmark/") { plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv", csvSeries: series, group: "${group.name}", title: "${title}", style: 'lineSimple', exclZero: false, keepRecords: false, logarithmic: !relative && group.log, numBuilds: '120', useDescr: true, yaxis: group.unit, yaxisMaximum: '', yaxisMinimum: '' } }