source: Jenkinsfile@ 21923bd

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 21923bd was c862fae, checked in by Thierry Delisle <tdelisle@…>, 6 years ago

Fixed typo in Jenkinsfile and added abort test to test libcfa archival
  • Property mode set to 100644
File size: 16.9 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[8ecb590]5// For skipping stages
6import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
7
[29f4fe62]8//===========================================================================================================
[9beae23]9// Main loop of the compilation
[29f4fe62]10//===========================================================================================================
[56a9ce6]11
[85142648]12node('master') {
13 // Globals
14 BuildDir = pwd tmp: true
15 SrcDir = pwd tmp: false
16 Settings = null
17 StageName = ''
[4f9e706]18
[85142648]19 // Local variables
20 def err = null
21 def log_needed = false
[5b8413b4]22
[85142648]23 currentBuild.result = "SUCCESS"
[4f9e706]24
[85142648]25 try {
26 //Wrap build to add timestamp to command line
27 wrap([$class: 'TimestamperBuildWrapper']) {
[0ef06b6]28
[85142648]29 Settings = prepare_build()
[952ee7a]30
[85142648]31 node(Settings.Architecture.node) {
32 BuildDir = pwd tmp: true
33 SrcDir = pwd tmp: false
[23a14d86]34
[85142648]35 clean()
[7aebc62]36
[85142648]37 checkout()
[fde808df]38
[85142648]39 build()
[95fdb0a]40
[85142648]41 test()
[95fdb0a]42
[85142648]43 benchmark()
[95fdb0a]44
[85142648]45 build_doc()
[7359098]46
[85142648]47 publish()
48 }
[9beae23]49
[85142648]50 // Update the build directories when exiting the node
51 BuildDir = pwd tmp: true
52 SrcDir = pwd tmp: false
[9beae23]53 }
[738cf8f]54 }
55
[85142648]56 //If an exception is caught we need to change the status and remember to
57 //attach the build log to the email
58 catch (Exception caughtError) {
59 //rethrow error later
60 err = caughtError
[738cf8f]61
[85142648]62 echo err.toString()
[e966ec0]63
[85142648]64 //An error has occured, the build log is relevent
65 log_needed = true
[9beae23]66
[85142648]67 //Store the result of the build log
68 currentBuild.result = "${StageName} FAILURE".trim()
69 }
[738cf8f]70
[85142648]71 finally {
72 //Send email with final results if this is not a full build
73 email(log_needed)
[9beae23]74
[85142648]75 echo 'Build Completed'
[734891d]76
[85142648]77 /* Must re-throw exception to propagate error */
78 if (err) {
79 throw err
80 }
[738cf8f]81 }
82}
[29f4fe62]83//===========================================================================================================
[9beae23]84// Main compilation routines
[29f4fe62]85//===========================================================================================================
[0dc3ac3]86def clean() {
[6c55a3d]87 build_stage('Cleanup', true) {
[0dc3ac3]88 // clean the build by wipping the build directory
[5b8413b4]89 dir(BuildDir) {
[d4cd491]90 deleteDir()
[ece8a80]91 }
[620dd2b]92 }
[95fdb0a]93}
[29f4fe62]94
[0dc3ac3]95//Compilation script is done here but environnement set-up and error handling is done in main loop
96def checkout() {
[6c55a3d]97 build_stage('Checkout', true) {
[0dc3ac3]98 //checkout the source code and clean the repo
[a336d46]99 final scmVars = checkout scm
100 Settings.GitNewRef = scmVars.GIT_COMMIT
101 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
[d3a4564a]102
[a336d46]103 echo GitLogMessage()
[e11957e]104
105 // This is a complete hack but it solves problems with automake thinking it needs to regenerate makefiles
106 // We fudged automake/missing to handle that but automake stills bakes prints inside the makefiles
107 // and these cause more problems.
108 sh 'find . -name Makefile.in -exec touch {} +'
[0dc3ac3]109 }
110}
111
[95fdb0a]112def build() {
[e507c11]113 debug = true
114 release = Settings.RunAllTests || Settings.RunBenchmark
115 build_stage('Build : configure', true) {
[50f2cfc]116 // Build outside of the src tree to ease cleaning
[5b8413b4]117 dir (BuildDir) {
[50f2cfc]118 //Configure the conpilation (Output is not relevant)
119 //Use the current directory as the installation target so nothing escapes the sandbox
120 //Also specify the compiler by hand
[3fc5f010]121 targets=""
[d4510ea]122 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]123 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]124 } else {
[6bde81d]125 targets="--with-target-hosts='host:debug'"
[3fc5f010]126 }
127
[aa96fba]128 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} --quiet --prefix=${BuildDir}"
[f253e4a]129
130 // Configure libcfa
[e70e54e]131 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]132 }
133 }
134
135 build_stage('Build : cfa-cpp', true) {
136 // Build outside of the src tree to ease cleaning
137 dir (BuildDir) {
138 // Build driver
139 sh 'make -j 8 --no-print-directory -C driver'
140
141 // Build translator
142 sh 'make -j 8 --no-print-directory -C src'
143 }
144 }
[1752d0e]145
[e507c11]146 build_stage('Build : libcfa(debug)', debug) {
147 // Build outside of the src tree to ease cleaning
148 dir (BuildDir) {
149 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
150 }
151 }
152
153 build_stage('Build : libcfa(nodebug)', release) {
154 // Build outside of the src tree to ease cleaning
155 dir (BuildDir) {
156 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]157 }
[620dd2b]158 }
[aa96fba]159
160 build_stage('Build : install', true) {
161 // Build outside of the src tree to ease cleaning
162 dir (BuildDir) {
163 sh "make -j 8 --no-print-directory install"
164 }
165 }
[95fdb0a]166}
[24eecab]167
[95fdb0a]168def test() {
[4c1b9ea8]169 try {
170 build_stage('Test: short', !Settings.RunAllTests) {
171 dir (BuildDir) {
[3e93c00]172 //Run the tests from the tests directory
[4c51aca]173 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]174 }
[4c1b9ea8]175 }
176
177 build_stage('Test: full', Settings.RunAllTests) {
178 dir (BuildDir) {
179 //Run the tests from the tests directory
[4c51aca]180 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
181 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]182 }
[9beae23]183 }
[620dd2b]184 }
[4c1b9ea8]185 catch (Exception err) {
186 echo "Archiving core dumps"
[c95fdc9]187 dir (BuildDir) {
[62f96ae]188 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
[c95fdc9]189 }
[4c1b9ea8]190 throw err
191 }
[95fdb0a]192}
193
194def benchmark() {
[6c55a3d]195 build_stage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]196 dir (BuildDir) {
[0dc3ac3]197 //Append bench results
[c6f1f3e]198 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]199 }
[620dd2b]200 }
[9beae23]201}
[efd60d67]202
[95fdb0a]203def build_doc() {
[6c55a3d]204 build_stage('Documentation', Settings.BuildDocumentation) {
[9beae23]205 dir ('doc/user') {
206 make_doc()
207 }
208
209 dir ('doc/refrat') {
210 make_doc()
211 }
[620dd2b]212 }
[9beae23]213}
214
[95fdb0a]215def publish() {
[6c55a3d]216 build_stage('Publish', true) {
[95fdb0a]217
[1b3eef8]218 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]219
[3221a2b]220 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
221 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]222
[3898392]223 //Then publish the results
[13d2dac]224 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
225 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
226 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]232 }
[95fdb0a]233}
234
[29f4fe62]235//===========================================================================================================
236//Routine responsible of sending the email notification once the build is completed
237//===========================================================================================================
[6b6c26e]238@NonCPS
239def SplitLines(String text) {
240 def list = []
241
242 text.eachLine {
243 list += it
244 }
245
246 return list
247}
248
[a336d46]249def GitLogMessage() {
250 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
[e8a22a7]251
[fce01e7]252 def oldRef = Settings.GitOldRef
253 def newRef = Settings.GitNewRef
[6badd87]254
[fdb6ac6]255 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
[6b6c26e]256 def revList = SplitLines( revText )
[6e31c43]257
[fdb6ac6]258 def gitUpdate = ""
259 revList.each { rev ->
[249091f]260 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
[fce01e7]261 gitUpdate = gitUpdate + " via ${rev} (${type})"
262 }
263
264 def rev = oldRef
[249091f]265 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
266 gitUpdate = gitUpdate + " from ${rev} (${type})"
[fce01e7]267
[249091f]268 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
[fce01e7]269
[249091f]270 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
[fce01e7]271 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
272 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
273 gitDiff = gitDiff.replace('[m', '</span>')
[7a927ed0]274
[a336d46]275 return """
[13c98a4]276<pre>
[848fb00]277The branch ${env.BRANCH_NAME} has been updated.
[7a927ed0]278${gitUpdate}
[13c98a4]279</pre>
280
281<p>Check console output at ${env.BUILD_URL} to view the results.</p>
[7b1a604]282
[13c98a4]283<p>- Status --------------------------------------------------------------</p>
[7b1a604]284
[13c98a4]285<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
[7b1a604]286
[13c98a4]287<p>- Log -----------------------------------------------------------------</p>
[e8a22a7]288
[13c98a4]289<pre>
[7a927ed0]290${gitLog}
[13c98a4]291</pre>
292
293<p>-----------------------------------------------------------------------</p>
294<pre>
[7b1a604]295Summary of changes:
[7a927ed0]296${gitDiff}
[13c98a4]297</pre>
[7b1a604]298"""
[a336d46]299}
300
301//Standard build email notification
[13c98a4]302def email(boolean log) {
[a336d46]303 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
304 //Configurations for email format
305 echo 'Notifying users of result'
306
307 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
308 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]309 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]310generated because of a git hooks/post-receive script following
[986e260]311a ref change which was pushed to the C\u2200 repository.</p>
[a336d46]312""" + GitLogMessage()
[e8a22a7]313
[13c98a4]314 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]315
[13c98a4]316 if( Settings && !Settings.Silent ) {
[094a42c]317 //send email notification
318 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
319 } else {
320 echo "Would send email to: ${email_to}"
321 echo "With title: ${email_subject}"
322 echo "Content: \n${email_body}"
323 }
[e8a22a7]324}
[5b8413b4]325
326//===========================================================================================================
327// Helper classes/variables/routines
328//===========================================================================================================
329//Description of a compiler (Must be serializable since pipelines are persistent)
330class CC_Desc implements Serializable {
[93fe3154]331 public String name
332 public String CXX
[6ebc13f]333 public String CC
[bf22bc6]334 public String lto
[93fe3154]335
[bf22bc6]336 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]337 this.name = name
338 this.CXX = CXX
[bf22bc6]339 this.CC = CC
340 this.lto = lto
[5b8413b4]341 }
342}
343
344//Description of an architecture (Must be serializable since pipelines are persistent)
345class Arch_Desc implements Serializable {
346 public String name
347 public String flags
[5307c33]348 public String node
[5b8413b4]349
[5307c33]350 Arch_Desc(String name, String flags, String node) {
[5b8413b4]351 this.name = name
352 this.flags = flags
[5307c33]353 this.node = node
[5b8413b4]354 }
355}
356
357class BuildSettings implements Serializable {
358 public final CC_Desc Compiler
359 public final Arch_Desc Architecture
360 public final Boolean RunAllTests
361 public final Boolean RunBenchmark
362 public final Boolean BuildDocumentation
363 public final Boolean Publish
364 public final Boolean Silent
365 public final Boolean IsSandbox
366 public final String DescLong
367 public final String DescShort
368
[a336d46]369 public String GitNewRef
370 public String GitOldRef
371
[490cb3c]372 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]373 switch( param.Compiler ) {
[099f5bd]374 case 'gcc-9':
[bf22bc6]375 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]376 break
377 case 'gcc-8':
[bf22bc6]378 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]379 break
380 case 'gcc-7':
[bf22bc6]381 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]382 break
[5b8413b4]383 case 'gcc-6':
[bf22bc6]384 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]385 break
386 case 'gcc-5':
[bf22bc6]387 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]388 break
389 case 'gcc-4.9':
[bf22bc6]390 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]391 break
392 case 'clang':
[bf22bc6]393 this.Compiler = new CC_Desc('clang', 'clang++-6.0', 'gcc-6', '-flto=thin -flto-jobs=0')
[5b8413b4]394 break
395 default :
396 error "Unhandled compiler : ${cc}"
397 }
398
399 switch( param.Architecture ) {
400 case 'x64':
[a3e8281]401 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]402 break
403 case 'x86':
[a3e8281]404 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]405 break
406 default :
407 error "Unhandled architecture : ${arch}"
408 }
409
[f95e8f0]410 this.IsSandbox = (branch == "jenkins-sandbox")
[5b8413b4]411 this.RunAllTests = param.RunAllTests
[7a230fd]412 this.RunBenchmark = param.RunBenchmark
[5b8413b4]413 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]414 this.Publish = param.Publish
[5b8413b4]415 this.Silent = param.Silent
416
417 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]418 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]419
[93fe3154]420 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[5b8413b4]421Architecture : ${ this.Architecture.name }
422Arc Flags : ${ this.Architecture.flags }
423Run All Tests : ${ this.RunAllTests.toString() }
424Run Benchmark : ${ this.RunBenchmark.toString() }
425Build Documentation : ${ this.BuildDocumentation.toString() }
426Publish : ${ this.Publish.toString() }
427Silent : ${ this.Silent.toString() }
428"""
[a336d46]429
430 this.GitNewRef = ''
431 this.GitOldRef = ''
[5b8413b4]432 }
433}
434
[490cb3c]435class PlotGroup implements Serializable {
436 public String name
437 public String unit
438 public boolean log
439
440 PlotGroup(String name, String unit, boolean log) {
441 this.name = name
442 this.unit = unit
443 this.log = log
444 }
445}
446
[5b8413b4]447def prepare_build() {
448 // prepare the properties
449 properties ([ \
[62f96ae]450 buildDiscarder(logRotator( \
451 artifactDaysToKeepStr: '', \
452 artifactNumToKeepStr: '', \
453 daysToKeepStr: '730', \
454 numToKeepStr: '1000' \
455 )), \
[5b8413b4]456 [$class: 'ParametersDefinitionProperty', \
457 parameterDefinitions: [ \
458 [$class: 'ChoiceParameterDefinition', \
459 description: 'Which compiler to use', \
460 name: 'Compiler', \
[fe27d99]461 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
462 defaultValue: 'gcc-8', \
[5b8413b4]463 ], \
464 [$class: 'ChoiceParameterDefinition', \
465 description: 'The target architecture', \
466 name: 'Architecture', \
467 choices: 'x64\nx86', \
468 defaultValue: 'x64', \
469 ], \
470 [$class: 'BooleanParameterDefinition', \
471 description: 'If false, only the quick test suite is ran', \
472 name: 'RunAllTests', \
473 defaultValue: false, \
474 ], \
475 [$class: 'BooleanParameterDefinition', \
476 description: 'If true, jenkins also runs benchmarks', \
477 name: 'RunBenchmark', \
478 defaultValue: false, \
479 ], \
480 [$class: 'BooleanParameterDefinition', \
481 description: 'If true, jenkins also builds documentation', \
482 name: 'BuildDocumentation', \
483 defaultValue: true, \
484 ], \
485 [$class: 'BooleanParameterDefinition', \
486 description: 'If true, jenkins also publishes results', \
487 name: 'Publish', \
488 defaultValue: false, \
489 ], \
490 [$class: 'BooleanParameterDefinition', \
491 description: 'If true, jenkins will not send emails', \
492 name: 'Silent', \
493 defaultValue: false, \
494 ], \
495 ],
496 ]])
497
[4c55047]498 // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer
499 checkout scm
[2407853]500
[490cb3c]501 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]502
503 currentBuild.description = settings.DescShort
504 echo settings.DescLong
505
506 return settings
507}
508
[6c55a3d]509def build_stage(String name, boolean run, Closure block ) {
[5b8413b4]510 StageName = name
511 echo " -------- ${StageName} -------- "
[8ecb590]512 if(run) {
513 stage(name, block)
514 } else {
515 stage(name) { Utils.markStageSkippedForConditional(STAGE_NAME) }
516 }
[5b8413b4]517}
518
519def make_doc() {
520 def err = null
521 try {
522 sh 'make clean > /dev/null'
523 sh 'make > /dev/null 2>&1'
524 }
525 catch (Exception caughtError) {
526 err = caughtError //rethrow error later
[65f4a51]527 sh 'cat build/*.log'
[5b8413b4]528 }
529 finally {
530 if (err) throw err // Must re-throw exception to propagate error
531 }
[a2a0065]532}
533
[3221a2b]534def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]535
[1b3eef8]536 if(new_data) {
537 echo "Publishing new data"
538 }
539
[cdcd53dc]540 def series = new_data ? [[
[df57a84]541 file: "${file}.csv",
[3c40dc2a]542 exclusionValues: '',
543 displayTableFlag: false,
544 inclusionFlag: 'OFF',
545 url: ''
[cdcd53dc]546 ]] : [];
[8d63649]547
548 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
549 dir("${BuildDir}/benchmark/") {
550 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
551 csvSeries: series,
[490cb3c]552 group: "${group.name}",
[3c40dc2a]553 title: "${title}",
554 style: 'lineSimple',
555 exclZero: false,
556 keepRecords: false,
[3221a2b]557 logarithmic: !relative && group.log,
[3c40dc2a]558 numBuilds: '120',
559 useDescr: true,
[490cb3c]560 yaxis: group.unit,
[3c40dc2a]561 yaxisMaximum: '',
562 yaxisMinimum: ''
[df57a84]563 }
564}
Note: See TracBrowser for help on using the repository browser.