source: Jenkinsfile@ fe3d9ab

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since fe3d9ab was fe3d9ab, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Jenkins now prints AST version in long description
  • Property mode set to 100644
File size: 17.0 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[29f4fe62]5//===========================================================================================================
[9beae23]6// Main loop of the compilation
[29f4fe62]7//===========================================================================================================
[56a9ce6]8
[85142648]9node('master') {
10 // Globals
11 BuildDir = pwd tmp: true
12 SrcDir = pwd tmp: false
13 Settings = null
[1483a16]14 Tools = null
[4f9e706]15
[85142648]16 // Local variables
17 def err = null
18 def log_needed = false
[5b8413b4]19
[85142648]20 currentBuild.result = "SUCCESS"
[4f9e706]21
[85142648]22 try {
23 //Wrap build to add timestamp to command line
24 wrap([$class: 'TimestamperBuildWrapper']) {
[0ef06b6]25
[85142648]26 Settings = prepare_build()
[952ee7a]27
[85142648]28 node(Settings.Architecture.node) {
29 BuildDir = pwd tmp: true
30 SrcDir = pwd tmp: false
[23a14d86]31
[85142648]32 clean()
[7aebc62]33
[85142648]34 checkout()
[fde808df]35
[85142648]36 build()
[95fdb0a]37
[85142648]38 test()
[95fdb0a]39
[85142648]40 benchmark()
[95fdb0a]41
[85142648]42 build_doc()
[7359098]43
[85142648]44 publish()
45 }
[9beae23]46
[85142648]47 // Update the build directories when exiting the node
48 BuildDir = pwd tmp: true
49 SrcDir = pwd tmp: false
[9beae23]50 }
[738cf8f]51 }
52
[85142648]53 //If an exception is caught we need to change the status and remember to
54 //attach the build log to the email
55 catch (Exception caughtError) {
56 //rethrow error later
57 err = caughtError
[738cf8f]58
[85142648]59 echo err.toString()
[e966ec0]60
[85142648]61 //An error has occured, the build log is relevent
62 log_needed = true
[9beae23]63
[85142648]64 //Store the result of the build log
[27b1ca1]65 currentBuild.result = "${tools.StageName} FAILURE".trim()
[85142648]66 }
[738cf8f]67
[85142648]68 finally {
69 //Send email with final results if this is not a full build
70 email(log_needed)
[9beae23]71
[85142648]72 echo 'Build Completed'
[734891d]73
[85142648]74 /* Must re-throw exception to propagate error */
75 if (err) {
76 throw err
77 }
[738cf8f]78 }
79}
[29f4fe62]80//===========================================================================================================
[9beae23]81// Main compilation routines
[29f4fe62]82//===========================================================================================================
[0dc3ac3]83def clean() {
[91aa5ab]84 Tools.BuildStage('Cleanup', true) {
[0dc3ac3]85 // clean the build by wipping the build directory
[5b8413b4]86 dir(BuildDir) {
[d4cd491]87 deleteDir()
[ece8a80]88 }
[620dd2b]89 }
[95fdb0a]90}
[29f4fe62]91
[0dc3ac3]92//Compilation script is done here but environnement set-up and error handling is done in main loop
93def checkout() {
[91aa5ab]94 Tools.BuildStage('Checkout', true) {
[0dc3ac3]95 //checkout the source code and clean the repo
[a336d46]96 final scmVars = checkout scm
97 Settings.GitNewRef = scmVars.GIT_COMMIT
98 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
[d3a4564a]99
[a336d46]100 echo GitLogMessage()
[0dc3ac3]101 }
102}
103
[95fdb0a]104def build() {
[e507c11]105 debug = true
106 release = Settings.RunAllTests || Settings.RunBenchmark
[91aa5ab]107 Tools.BuildStage('Build : configure', true) {
[dcf1979]108 // Configure must be run inside the tree
109 dir (SrcDir) {
110 // Generate the necessary build files
111 sh './autogen.sh'
112 }
113
[50f2cfc]114 // Build outside of the src tree to ease cleaning
[5b8413b4]115 dir (BuildDir) {
[50f2cfc]116 //Configure the conpilation (Output is not relevant)
117 //Use the current directory as the installation target so nothing escapes the sandbox
118 //Also specify the compiler by hand
[3fc5f010]119 targets=""
[d4510ea]120 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]121 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]122 } else {
[6bde81d]123 targets="--with-target-hosts='host:debug'"
[3fc5f010]124 }
125
[d21dd3cb]126 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
127
128 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]129
130 // Configure libcfa
[e70e54e]131 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]132 }
133 }
134
[91aa5ab]135 Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]136 // Build outside of the src tree to ease cleaning
137 dir (BuildDir) {
138 // Build driver
139 sh 'make -j 8 --no-print-directory -C driver'
140
141 // Build translator
142 sh 'make -j 8 --no-print-directory -C src'
143 }
144 }
[1752d0e]145
[91aa5ab]146 Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]147 // Build outside of the src tree to ease cleaning
148 dir (BuildDir) {
149 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
150 }
151 }
152
[91aa5ab]153 Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]154 // Build outside of the src tree to ease cleaning
155 dir (BuildDir) {
156 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]157 }
[620dd2b]158 }
[aa96fba]159
[91aa5ab]160 Tools.BuildStage('Build : install', true) {
[aa96fba]161 // Build outside of the src tree to ease cleaning
162 dir (BuildDir) {
163 sh "make -j 8 --no-print-directory install"
164 }
165 }
[95fdb0a]166}
[24eecab]167
[95fdb0a]168def test() {
[4c1b9ea8]169 try {
[91aa5ab]170 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]171 dir (BuildDir) {
[3e93c00]172 //Run the tests from the tests directory
[4c51aca]173 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]174 }
[4c1b9ea8]175 }
176
[91aa5ab]177 Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]178 dir (BuildDir) {
179 //Run the tests from the tests directory
[4c51aca]180 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
181 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]182 }
[9beae23]183 }
[620dd2b]184 }
[4c1b9ea8]185 catch (Exception err) {
186 echo "Archiving core dumps"
[c95fdc9]187 dir (BuildDir) {
[62f96ae]188 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
[c95fdc9]189 }
[4c1b9ea8]190 throw err
191 }
[95fdb0a]192}
193
194def benchmark() {
[91aa5ab]195 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]196 dir (BuildDir) {
[0dc3ac3]197 //Append bench results
[c6f1f3e]198 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]199 }
[620dd2b]200 }
[9beae23]201}
[efd60d67]202
[95fdb0a]203def build_doc() {
[91aa5ab]204 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]205 dir ('doc/user') {
206 make_doc()
207 }
208
209 dir ('doc/refrat') {
210 make_doc()
211 }
[620dd2b]212 }
[9beae23]213}
214
[95fdb0a]215def publish() {
[91aa5ab]216 Tools.BuildStage('Publish', true) {
[95fdb0a]217
[1b3eef8]218 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]219
[3221a2b]220 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
221 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]222
[3898392]223 //Then publish the results
[13d2dac]224 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
225 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
226 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]232 }
[95fdb0a]233}
234
[29f4fe62]235//===========================================================================================================
236//Routine responsible of sending the email notification once the build is completed
237//===========================================================================================================
[6b6c26e]238@NonCPS
239def SplitLines(String text) {
240 def list = []
241
242 text.eachLine {
243 list += it
244 }
245
246 return list
247}
248
[a336d46]249def GitLogMessage() {
250 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
[e8a22a7]251
[fce01e7]252 def oldRef = Settings.GitOldRef
253 def newRef = Settings.GitNewRef
[6badd87]254
[fdb6ac6]255 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
[6b6c26e]256 def revList = SplitLines( revText )
[6e31c43]257
[fdb6ac6]258 def gitUpdate = ""
259 revList.each { rev ->
[249091f]260 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
[fce01e7]261 gitUpdate = gitUpdate + " via ${rev} (${type})"
262 }
263
264 def rev = oldRef
[249091f]265 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
266 gitUpdate = gitUpdate + " from ${rev} (${type})"
[fce01e7]267
[249091f]268 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
[fce01e7]269
[249091f]270 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
[fce01e7]271 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
272 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
273 gitDiff = gitDiff.replace('[m', '</span>')
[7a927ed0]274
[a336d46]275 return """
[13c98a4]276<pre>
[848fb00]277The branch ${env.BRANCH_NAME} has been updated.
[7a927ed0]278${gitUpdate}
[13c98a4]279</pre>
280
281<p>Check console output at ${env.BUILD_URL} to view the results.</p>
[7b1a604]282
[13c98a4]283<p>- Status --------------------------------------------------------------</p>
[7b1a604]284
[13c98a4]285<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
[7b1a604]286
[13c98a4]287<p>- Log -----------------------------------------------------------------</p>
[e8a22a7]288
[13c98a4]289<pre>
[7a927ed0]290${gitLog}
[13c98a4]291</pre>
292
293<p>-----------------------------------------------------------------------</p>
294<pre>
[7b1a604]295Summary of changes:
[7a927ed0]296${gitDiff}
[13c98a4]297</pre>
[7b1a604]298"""
[a336d46]299}
300
301//Standard build email notification
[13c98a4]302def email(boolean log) {
[a336d46]303 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
304 //Configurations for email format
305 echo 'Notifying users of result'
306
307 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
308 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]309 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]310generated because of a git hooks/post-receive script following
[986e260]311a ref change which was pushed to the C\u2200 repository.</p>
[a336d46]312""" + GitLogMessage()
[e8a22a7]313
[13c98a4]314 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]315
[13c98a4]316 if( Settings && !Settings.Silent ) {
[094a42c]317 //send email notification
318 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
319 } else {
320 echo "Would send email to: ${email_to}"
321 echo "With title: ${email_subject}"
322 echo "Content: \n${email_body}"
323 }
[e8a22a7]324}
[5b8413b4]325
326//===========================================================================================================
327// Helper classes/variables/routines
328//===========================================================================================================
329//Description of a compiler (Must be serializable since pipelines are persistent)
330class CC_Desc implements Serializable {
[93fe3154]331 public String name
332 public String CXX
[6ebc13f]333 public String CC
[bf22bc6]334 public String lto
[93fe3154]335
[bf22bc6]336 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]337 this.name = name
338 this.CXX = CXX
[bf22bc6]339 this.CC = CC
340 this.lto = lto
[5b8413b4]341 }
342}
343
344//Description of an architecture (Must be serializable since pipelines are persistent)
345class Arch_Desc implements Serializable {
346 public String name
347 public String flags
[5307c33]348 public String node
[5b8413b4]349
[5307c33]350 Arch_Desc(String name, String flags, String node) {
[5b8413b4]351 this.name = name
352 this.flags = flags
[5307c33]353 this.node = node
[5b8413b4]354 }
355}
356
357class BuildSettings implements Serializable {
358 public final CC_Desc Compiler
359 public final Arch_Desc Architecture
[d21dd3cb]360 public final Boolean NewAST
[5b8413b4]361 public final Boolean RunAllTests
362 public final Boolean RunBenchmark
363 public final Boolean BuildDocumentation
364 public final Boolean Publish
365 public final Boolean Silent
366 public final Boolean IsSandbox
367 public final String DescLong
368 public final String DescShort
369
[a336d46]370 public String GitNewRef
371 public String GitOldRef
372
[490cb3c]373 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]374 switch( param.Compiler ) {
[099f5bd]375 case 'gcc-9':
[bf22bc6]376 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]377 break
378 case 'gcc-8':
[bf22bc6]379 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]380 break
381 case 'gcc-7':
[bf22bc6]382 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]383 break
[5b8413b4]384 case 'gcc-6':
[bf22bc6]385 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]386 break
387 case 'gcc-5':
[bf22bc6]388 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]389 break
390 case 'gcc-4.9':
[bf22bc6]391 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]392 break
393 case 'clang':
[391c065]394 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]395 break
396 default :
397 error "Unhandled compiler : ${cc}"
398 }
399
400 switch( param.Architecture ) {
401 case 'x64':
[a3e8281]402 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]403 break
404 case 'x86':
[a3e8281]405 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]406 break
407 default :
408 error "Unhandled architecture : ${arch}"
409 }
410
[f95e8f0]411 this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]412 this.NewAST = param.NewAST
[5b8413b4]413 this.RunAllTests = param.RunAllTests
[7a230fd]414 this.RunBenchmark = param.RunBenchmark
[5b8413b4]415 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]416 this.Publish = param.Publish
[5b8413b4]417 this.Silent = param.Silent
418
419 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]420 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]421
[fe3d9ab]422 final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]423 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[fe3d9ab]424AST Version : ${ this.RunAllTests.toString() }
[5b8413b4]425Architecture : ${ this.Architecture.name }
426Arc Flags : ${ this.Architecture.flags }
427Run All Tests : ${ this.RunAllTests.toString() }
428Run Benchmark : ${ this.RunBenchmark.toString() }
429Build Documentation : ${ this.BuildDocumentation.toString() }
430Publish : ${ this.Publish.toString() }
431Silent : ${ this.Silent.toString() }
432"""
[a336d46]433
434 this.GitNewRef = ''
435 this.GitOldRef = ''
[5b8413b4]436 }
437}
438
[490cb3c]439class PlotGroup implements Serializable {
440 public String name
441 public String unit
442 public boolean log
443
444 PlotGroup(String name, String unit, boolean log) {
445 this.name = name
446 this.unit = unit
447 this.log = log
448 }
449}
450
[5b8413b4]451def prepare_build() {
452 // prepare the properties
453 properties ([ \
[62f96ae]454 buildDiscarder(logRotator( \
455 artifactDaysToKeepStr: '', \
456 artifactNumToKeepStr: '', \
457 daysToKeepStr: '730', \
458 numToKeepStr: '1000' \
459 )), \
[5b8413b4]460 [$class: 'ParametersDefinitionProperty', \
461 parameterDefinitions: [ \
462 [$class: 'ChoiceParameterDefinition', \
463 description: 'Which compiler to use', \
464 name: 'Compiler', \
[c09ae73]465 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]466 defaultValue: 'gcc-8', \
[5b8413b4]467 ], \
468 [$class: 'ChoiceParameterDefinition', \
469 description: 'The target architecture', \
470 name: 'Architecture', \
471 choices: 'x64\nx86', \
472 defaultValue: 'x64', \
473 ], \
[d21dd3cb]474 [$class: 'BooleanParameterDefinition', \
475 description: 'If true, build compiler using new AST', \
476 name: 'NewAST', \
477 defaultValue: false, \
478 ], \
[5b8413b4]479 [$class: 'BooleanParameterDefinition', \
480 description: 'If false, only the quick test suite is ran', \
481 name: 'RunAllTests', \
482 defaultValue: false, \
[fe3d9ab]483 ], \
[5b8413b4]484 [$class: 'BooleanParameterDefinition', \
485 description: 'If true, jenkins also runs benchmarks', \
486 name: 'RunBenchmark', \
487 defaultValue: false, \
488 ], \
489 [$class: 'BooleanParameterDefinition', \
490 description: 'If true, jenkins also builds documentation', \
491 name: 'BuildDocumentation', \
492 defaultValue: true, \
493 ], \
494 [$class: 'BooleanParameterDefinition', \
495 description: 'If true, jenkins also publishes results', \
496 name: 'Publish', \
497 defaultValue: false, \
498 ], \
499 [$class: 'BooleanParameterDefinition', \
500 description: 'If true, jenkins will not send emails', \
501 name: 'Silent', \
502 defaultValue: false, \
503 ], \
504 ],
505 ]])
506
[bd8dca2]507 // It's unfortunate but it looks like we need to checkout the entire repo just to get
508 // - the pretty git printer
509 // - Jenkins.tools
[4c55047]510 checkout scm
[2407853]511
[1483a16]512 Tools = load "Jenkins/tools.groovy"
[bd8dca2]513
[490cb3c]514 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]515
516 currentBuild.description = settings.DescShort
517 echo settings.DescLong
518
519 return settings
520}
521
522def make_doc() {
523 def err = null
524 try {
525 sh 'make clean > /dev/null'
526 sh 'make > /dev/null 2>&1'
527 }
528 catch (Exception caughtError) {
529 err = caughtError //rethrow error later
[65f4a51]530 sh 'cat build/*.log'
[5b8413b4]531 }
532 finally {
533 if (err) throw err // Must re-throw exception to propagate error
534 }
[a2a0065]535}
536
[3221a2b]537def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]538
[1b3eef8]539 if(new_data) {
540 echo "Publishing new data"
541 }
542
[cdcd53dc]543 def series = new_data ? [[
[df57a84]544 file: "${file}.csv",
[3c40dc2a]545 exclusionValues: '',
546 displayTableFlag: false,
547 inclusionFlag: 'OFF',
548 url: ''
[cdcd53dc]549 ]] : [];
[8d63649]550
551 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
552 dir("${BuildDir}/benchmark/") {
553 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
554 csvSeries: series,
[490cb3c]555 group: "${group.name}",
[3c40dc2a]556 title: "${title}",
557 style: 'lineSimple',
558 exclZero: false,
559 keepRecords: false,
[3221a2b]560 logarithmic: !relative && group.log,
[3c40dc2a]561 numBuilds: '120',
562 useDescr: true,
[490cb3c]563 yaxis: group.unit,
[3c40dc2a]564 yaxisMaximum: '',
565 yaxisMinimum: ''
[df57a84]566 }
567}
Note: See TracBrowser for help on using the repository browser.