source: Jenkinsfile @ 85799aa

#!groovy

import groovy.transform.Field

//===========================================================================================================
// Main loop of the compilation
//===========================================================================================================

node('master') {
        // Globals
        BuildDir  = pwd tmp: true
        SrcDir    = pwd tmp: false
        Settings  = null
        Tools     = null

        // Local variables
        def err = null
        def log_needed = false

        currentBuild.result = "SUCCESS"

        try {
                //Wrap build to add timestamp to command line
                wrap([$class: 'TimestamperBuildWrapper']) {

                        Settings = prepare_build()

                        node(Settings.Architecture.node) {
                                BuildDir  = pwd tmp: true
                                SrcDir    = pwd tmp: false

                                clean()

                                checkout()

                                build()

                                test()

                                benchmark()

                                build_doc()

                                publish()
                        }

                        // Update the build directories when exiting the node
                        BuildDir  = pwd tmp: true
                        SrcDir    = pwd tmp: false
                }
        }

        //If an exception is caught we need to change the status and remember to
        //attach the build log to the email
        catch (Exception caughtError) {
                //rethrow error later
                err = caughtError

                echo err.toString()

                //An error has occured, the build log is relevent
                log_needed = true

                //Store the result of the build log
                currentBuild.result = "${tools.StageName} FAILURE".trim()
        }

        finally {
                //Send email with final results if this is not a full build
                email(log_needed)

                echo 'Build Completed'

                /* Must re-throw exception to propagate error */
                if (err) {
                        throw err
                }
        }
}
//===========================================================================================================
// Main compilation routines
//===========================================================================================================
def clean() {
        Tools.BuildStage('Cleanup', true) {
                // clean the build by wipping the build directory
                dir(BuildDir) {
                        deleteDir()
                }
        }
}

//Compilation script is done here but environnement set-up and error handling is done in main loop
def checkout() {
        Tools.BuildStage('Checkout', true) {
                //checkout the source code and clean the repo
                final scmVars = checkout scm
                Settings.GitNewRef = scmVars.GIT_COMMIT
                Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT

                echo GitLogMessage()
        }
}

def build() {
        debug = true
        release = Settings.RunAllTests || Settings.RunBenchmark
        Tools.BuildStage('Build : configure', true) {
                // Configure must be run inside the tree
                dir (SrcDir) {
                        // Generate the necessary build files
                        sh './autogen.sh'
                }

                // Build outside of the src tree to ease cleaning
                dir (BuildDir) {
                        //Configure the conpilation (Output is not relevant)
                        //Use the current directory as the installation target so nothing escapes the sandbox
                        //Also specify the compiler by hand
                        targets=""
                        if( Settings.RunAllTests || Settings.RunBenchmark ) {
                                targets="--with-target-hosts='host:debug,host:nodebug'"
                        } else {
                                targets="--with-target-hosts='host:debug'"
                        }

                        ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"

                        sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"

                        // Configure libcfa
                        sh 'make -j 8 --no-print-directory configure-libcfa'
                }
        }

        Tools.BuildStage('Build : cfa-cpp', true) {
                // Build outside of the src tree to ease cleaning
                dir (BuildDir) {
                        // Build driver
                        sh 'make -j 8 --no-print-directory -C driver'

                        // Build translator
                        sh 'make -j 8 --no-print-directory -C src'
                }
        }

        Tools.BuildStage('Build : libcfa(debug)', debug) {
                // Build outside of the src tree to ease cleaning
                dir (BuildDir) {
                        sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
                }
        }

        Tools.BuildStage('Build : libcfa(nodebug)', release) {
                // Build outside of the src tree to ease cleaning
                dir (BuildDir) {
                        sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
                }
        }

        Tools.BuildStage('Build : install', true) {
                // Build outside of the src tree to ease cleaning
                dir (BuildDir) {
                        sh "make -j 8 --no-print-directory install"
                }
        }
}

def test() {
        try {
                Tools.BuildStage('Test: short', !Settings.RunAllTests) {
                        dir (BuildDir) {
                                //Run the tests from the tests directory
                                sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
                        }
                }

                Tools.BuildStage('Test: full', Settings.RunAllTests) {
                        dir (BuildDir) {
                                        //Run the tests from the tests directory
                                        sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
                                        sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no  archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
                        }
                }
        }
        catch (Exception err) {
                echo "Archiving core dumps"
                dir (BuildDir) {
                        archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
                }
                throw err
        }
}

def benchmark() {
        Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
                dir (BuildDir) {
                        //Append bench results
                        sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
                }
        }
}

def build_doc() {
        Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
                dir ('doc/user') {
                        make_doc()
                }

                dir ('doc/refrat') {
                        make_doc()
                }
        }
}

def publish() {
        Tools.BuildStage('Publish', true) {

                if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }

                def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
                def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)

                //Then publish the results
                do_plot(Settings.RunBenchmark && Settings.Publish, 'compile'        , groupCompile    , false, 'Compilation')
                do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff'   , groupCompile    , true , 'Compilation (relative)')
                do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch'      , groupConcurrency, false, 'Context Switching')
                do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
                do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex'          , groupConcurrency, false, 'Mutual Exclusion')
                do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff'     , groupConcurrency, true , 'Mutual Exclusion (relative)')
                do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling'     , groupConcurrency, false, 'Internal and External Scheduling')
                do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
        }
}

//===========================================================================================================
//Routine responsible of sending the email notification once the build is completed
//===========================================================================================================
@NonCPS
def SplitLines(String text) {
        def list = []

        text.eachLine {
                list += it
        }

        return list
}

def GitLogMessage() {
        if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"

        def oldRef = Settings.GitOldRef
        def newRef = Settings.GitNewRef

        def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
        def revList = SplitLines( revText )

        def gitUpdate = ""
        revList.each { rev ->
                def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
                gitUpdate = gitUpdate + "       via  ${rev} (${type})"
        }

        def rev = oldRef
        def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
        gitUpdate = gitUpdate + "      from  ${rev} (${type})"

        def gitLog    = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()

        def gitDiff   = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
        gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
        gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
        gitDiff = gitDiff.replace('[m', '</span>')

        return """
<pre>
The branch ${env.BRANCH_NAME} has been updated.
${gitUpdate}
</pre>

<p>Check console output at ${env.BUILD_URL} to view the results.</p>

<p>- Status --------------------------------------------------------------</p>

<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>

<p>- Log -----------------------------------------------------------------</p>

<pre>
${gitLog}
</pre>

<p>-----------------------------------------------------------------------</p>
<pre>
Summary of changes:
${gitDiff}
</pre>
"""
}

//Standard build email notification
def email(boolean log) {
        //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
        //Configurations for email format
        echo 'Notifying users of result'

        def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
        def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
        def email_body = """<p>This is an automated email from the Jenkins build machine. It was
generated because of a git hooks/post-receive script following
a ref change which was pushed to the C\u2200 repository.</p>
""" + GitLogMessage()

        def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"

        if( Settings && !Settings.Silent ) {
                //send email notification
                emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
        } else {
                echo "Would send email to: ${email_to}"
                echo "With title: ${email_subject}"
                echo "Content: \n${email_body}"
        }
}

//===========================================================================================================
// Helper classes/variables/routines
//===========================================================================================================
//Description of a compiler (Must be serializable since pipelines are persistent)
class CC_Desc implements Serializable {
        public String name
        public String CXX
        public String CC
        public String lto

        CC_Desc(String name, String CXX, String CC, String lto) {
                this.name = name
                this.CXX = CXX
                this.CC  = CC
                this.lto = lto
        }
}

//Description of an architecture (Must be serializable since pipelines are persistent)
class Arch_Desc implements Serializable {
        public String name
        public String flags
        public String node

        Arch_Desc(String name, String flags, String node) {
                this.name  = name
                this.flags = flags
                this.node  = node
        }
}

class BuildSettings implements Serializable {
        public final CC_Desc Compiler
        public final Arch_Desc Architecture
        public final Boolean NewAST
        public final Boolean RunAllTests
        public final Boolean RunBenchmark
        public final Boolean BuildDocumentation
        public final Boolean Publish
        public final Boolean Silent
        public final Boolean IsSandbox
        public final String DescLong
        public final String DescShort

        public String GitNewRef
        public String GitOldRef

        BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
                switch( param.Compiler ) {
                        case 'gcc-9':
                                this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
                        break
                        case 'gcc-8':
                                this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
                        break
                        case 'gcc-7':
                                this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
                        break
                        case 'gcc-6':
                                this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
                        break
                        case 'gcc-5':
                                this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
                        break
                        case 'gcc-4.9':
                                this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
                        break
                        case 'clang':
                                this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
                        break
                        default :
                                error "Unhandled compiler : ${cc}"
                }

                switch( param.Architecture ) {
                        case 'x64':
                                this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
                        break
                        case 'x86':
                                this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
                        break
                        default :
                                error "Unhandled architecture : ${arch}"
                }

                this.IsSandbox          = (branch == "jenkins-sandbox")
                this.NewAST             = param.NewAST
                this.RunAllTests        = param.RunAllTests
                this.RunBenchmark       = param.RunBenchmark
                this.BuildDocumentation = param.BuildDocumentation
                this.Publish            = param.Publish
                this.Silent             = param.Silent

                def full = param.RunAllTests ? " (Full)" : ""
                this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"

                final ast = this.NewAST ? "New AST" : "Old AST"
                this.DescLong = """Compiler              : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
AST Version             : ${ ast.toString() }
Architecture            : ${ this.Architecture.name }
Arc Flags               : ${ this.Architecture.flags }
Run All Tests           : ${ this.RunAllTests.toString() }
Run Benchmark           : ${ this.RunBenchmark.toString() }
Build Documentation     : ${ this.BuildDocumentation.toString() }
Publish                 : ${ this.Publish.toString() }
Silent                  : ${ this.Silent.toString() }
"""

                this.GitNewRef = ''
                this.GitOldRef = ''
        }
}

class PlotGroup implements Serializable {
        public String name
        public String unit
        public boolean log

        PlotGroup(String name, String unit, boolean log) {
                this.name = name
                this.unit = unit
                this.log = log
        }
}

def prepare_build() {
        // prepare the properties
        properties ([                                                                                                   \
                buildDiscarder(logRotator(                                                                              \
                        artifactDaysToKeepStr: '',                                                                      \
                        artifactNumToKeepStr: '',                                                                       \
                        daysToKeepStr: '730',                                                                           \
                        numToKeepStr: '1000'                                                                            \
                )),                                                                                                             \
                [$class: 'ParametersDefinitionProperty',                                                                \
                        parameterDefinitions: [                                                                         \
                                [$class: 'ChoiceParameterDefinition',                                           \
                                        description: 'Which compiler to use',                                   \
                                        name: 'Compiler',                                                                       \
                                        choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang',   \
                                        defaultValue: 'gcc-8',                                                          \
                                ],                                                                                              \
                                [$class: 'ChoiceParameterDefinition',                                           \
                                        description: 'The target architecture',                                 \
                                        name: 'Architecture',                                                           \
                                        choices: 'x64\nx86',                                                            \
                                        defaultValue: 'x64',                                                            \
                                ],                                                                                              \
                                [$class: 'BooleanParameterDefinition',                                                  \
                                        description: 'If true, build compiler using new AST',           \
                                        name: 'NewAST',                                                                         \
                                        defaultValue: false,                                                            \
                                ],                                                                                              \
                                [$class: 'BooleanParameterDefinition',                                                  \
                                        description: 'If false, only the quick test suite is ran',              \
                                        name: 'RunAllTests',                                                            \
                                        defaultValue: false,                                                            \
                                ],                                                                                              \
                                [$class: 'BooleanParameterDefinition',                                                  \
                                        description: 'If true, jenkins also runs benchmarks',           \
                                        name: 'RunBenchmark',                                                           \
                                        defaultValue: false,                                                            \
                                ],                                                                                              \
                                [$class: 'BooleanParameterDefinition',                                                  \
                                        description: 'If true, jenkins also builds documentation',              \
                                        name: 'BuildDocumentation',                                                     \
                                        defaultValue: true,                                                             \
                                ],                                                                                              \
                                [$class: 'BooleanParameterDefinition',                                                  \
                                        description: 'If true, jenkins also publishes results',                 \
                                        name: 'Publish',                                                                        \
                                        defaultValue: false,                                                            \
                                ],                                                                                              \
                                [$class: 'BooleanParameterDefinition',                                                  \
                                        description: 'If true, jenkins will not send emails',           \
                                        name: 'Silent',                                                                         \
                                        defaultValue: false,                                                            \
                                ],                                                                                              \
                        ],
                ]])

        // It's unfortunate but it looks like we need to checkout the entire repo just to get
        // - the pretty git printer
        // - Jenkins.tools
        checkout scm

        Tools = load "Jenkins/tools.groovy"

        final settings = new BuildSettings(params, env.BRANCH_NAME)

        currentBuild.description = settings.DescShort
        echo                       settings.DescLong

        return settings
}

def make_doc() {
        def err = null
        try {
                sh 'make clean > /dev/null'
                sh 'make > /dev/null 2>&1'
        }
        catch (Exception caughtError) {
                err = caughtError //rethrow error later
                sh 'cat build/*.log'
        }
        finally {
                if (err) throw err // Must re-throw exception to propagate error
        }
}

def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {

        if(new_data) {
                echo "Publishing new data"
        }

        def series = new_data ? [[
                                file: "${file}.csv",
                                exclusionValues: '',
                                displayTableFlag: false,
                                inclusionFlag: 'OFF',
                                url: ''
                        ]] : [];

        echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
        dir("${BuildDir}/benchmark/") {
                plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
                        csvSeries: series,
                        group: "${group.name}",
                        title: "${title}",
                        style: 'lineSimple',
                        exclZero: false,
                        keepRecords: false,
                        logarithmic: !relative && group.log,
                        numBuilds: '120',
                        useDescr: true,
                        yaxis: group.unit,
                        yaxisMaximum: '',
                        yaxisMinimum: ''
        }
}