source: Jenkinsfile@ 27b1ca1

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 27b1ca1 was 27b1ca1, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Modified Jenkinsfile to move out some of the helper for later reuse
  • Property mode set to 100644
File size: 16.8 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[27b1ca1]5import static Jenkins.tools.build_stage as build_stage
6import Jenkins.tools
[8ecb590]7
[29f4fe62]8//===========================================================================================================
[9beae23]9// Main loop of the compilation
[29f4fe62]10//===========================================================================================================
[56a9ce6]11
[85142648]12node('master') {
13 // Globals
14 BuildDir = pwd tmp: true
15 SrcDir = pwd tmp: false
16 Settings = null
[4f9e706]17
[85142648]18 // Local variables
19 def err = null
20 def log_needed = false
[5b8413b4]21
[85142648]22 currentBuild.result = "SUCCESS"
[4f9e706]23
[85142648]24 try {
25 //Wrap build to add timestamp to command line
26 wrap([$class: 'TimestamperBuildWrapper']) {
[0ef06b6]27
[85142648]28 Settings = prepare_build()
[952ee7a]29
[85142648]30 node(Settings.Architecture.node) {
31 BuildDir = pwd tmp: true
32 SrcDir = pwd tmp: false
[23a14d86]33
[85142648]34 clean()
[7aebc62]35
[85142648]36 checkout()
[fde808df]37
[85142648]38 build()
[95fdb0a]39
[85142648]40 test()
[95fdb0a]41
[85142648]42 benchmark()
[95fdb0a]43
[85142648]44 build_doc()
[7359098]45
[85142648]46 publish()
47 }
[9beae23]48
[85142648]49 // Update the build directories when exiting the node
50 BuildDir = pwd tmp: true
51 SrcDir = pwd tmp: false
[9beae23]52 }
[738cf8f]53 }
54
[85142648]55 //If an exception is caught we need to change the status and remember to
56 //attach the build log to the email
57 catch (Exception caughtError) {
58 //rethrow error later
59 err = caughtError
[738cf8f]60
[85142648]61 echo err.toString()
[e966ec0]62
[85142648]63 //An error has occured, the build log is relevent
64 log_needed = true
[9beae23]65
[85142648]66 //Store the result of the build log
[27b1ca1]67 currentBuild.result = "${tools.StageName} FAILURE".trim()
[85142648]68 }
[738cf8f]69
[85142648]70 finally {
71 //Send email with final results if this is not a full build
72 email(log_needed)
[9beae23]73
[85142648]74 echo 'Build Completed'
[734891d]75
[85142648]76 /* Must re-throw exception to propagate error */
77 if (err) {
78 throw err
79 }
[738cf8f]80 }
81}
[29f4fe62]82//===========================================================================================================
[9beae23]83// Main compilation routines
[29f4fe62]84//===========================================================================================================
[0dc3ac3]85def clean() {
[6c55a3d]86 build_stage('Cleanup', true) {
[0dc3ac3]87 // clean the build by wipping the build directory
[5b8413b4]88 dir(BuildDir) {
[d4cd491]89 deleteDir()
[ece8a80]90 }
[620dd2b]91 }
[95fdb0a]92}
[29f4fe62]93
[0dc3ac3]94//Compilation script is done here but environnement set-up and error handling is done in main loop
95def checkout() {
[6c55a3d]96 build_stage('Checkout', true) {
[0dc3ac3]97 //checkout the source code and clean the repo
[a336d46]98 final scmVars = checkout scm
99 Settings.GitNewRef = scmVars.GIT_COMMIT
100 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
[d3a4564a]101
[a336d46]102 echo GitLogMessage()
[0dc3ac3]103 }
104}
105
[95fdb0a]106def build() {
[e507c11]107 debug = true
108 release = Settings.RunAllTests || Settings.RunBenchmark
109 build_stage('Build : configure', true) {
[dcf1979]110 // Configure must be run inside the tree
111 dir (SrcDir) {
112 // Generate the necessary build files
113 sh './autogen.sh'
114 }
115
[50f2cfc]116 // Build outside of the src tree to ease cleaning
[5b8413b4]117 dir (BuildDir) {
[50f2cfc]118 //Configure the conpilation (Output is not relevant)
119 //Use the current directory as the installation target so nothing escapes the sandbox
120 //Also specify the compiler by hand
[3fc5f010]121 targets=""
[d4510ea]122 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]123 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]124 } else {
[6bde81d]125 targets="--with-target-hosts='host:debug'"
[3fc5f010]126 }
127
[d21dd3cb]128 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
129
130 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]131
132 // Configure libcfa
[e70e54e]133 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]134 }
135 }
136
137 build_stage('Build : cfa-cpp', true) {
138 // Build outside of the src tree to ease cleaning
139 dir (BuildDir) {
140 // Build driver
141 sh 'make -j 8 --no-print-directory -C driver'
142
143 // Build translator
144 sh 'make -j 8 --no-print-directory -C src'
145 }
146 }
[1752d0e]147
[e507c11]148 build_stage('Build : libcfa(debug)', debug) {
149 // Build outside of the src tree to ease cleaning
150 dir (BuildDir) {
151 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
152 }
153 }
154
155 build_stage('Build : libcfa(nodebug)', release) {
156 // Build outside of the src tree to ease cleaning
157 dir (BuildDir) {
158 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]159 }
[620dd2b]160 }
[aa96fba]161
162 build_stage('Build : install', true) {
163 // Build outside of the src tree to ease cleaning
164 dir (BuildDir) {
165 sh "make -j 8 --no-print-directory install"
166 }
167 }
[95fdb0a]168}
[24eecab]169
[95fdb0a]170def test() {
[4c1b9ea8]171 try {
172 build_stage('Test: short', !Settings.RunAllTests) {
173 dir (BuildDir) {
[3e93c00]174 //Run the tests from the tests directory
[4c51aca]175 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]176 }
[4c1b9ea8]177 }
178
179 build_stage('Test: full', Settings.RunAllTests) {
180 dir (BuildDir) {
181 //Run the tests from the tests directory
[4c51aca]182 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
183 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]184 }
[9beae23]185 }
[620dd2b]186 }
[4c1b9ea8]187 catch (Exception err) {
188 echo "Archiving core dumps"
[c95fdc9]189 dir (BuildDir) {
[62f96ae]190 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
[c95fdc9]191 }
[4c1b9ea8]192 throw err
193 }
[95fdb0a]194}
195
196def benchmark() {
[6c55a3d]197 build_stage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]198 dir (BuildDir) {
[0dc3ac3]199 //Append bench results
[c6f1f3e]200 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]201 }
[620dd2b]202 }
[9beae23]203}
[efd60d67]204
[95fdb0a]205def build_doc() {
[6c55a3d]206 build_stage('Documentation', Settings.BuildDocumentation) {
[9beae23]207 dir ('doc/user') {
208 make_doc()
209 }
210
211 dir ('doc/refrat') {
212 make_doc()
213 }
[620dd2b]214 }
[9beae23]215}
216
[95fdb0a]217def publish() {
[6c55a3d]218 build_stage('Publish', true) {
[95fdb0a]219
[1b3eef8]220 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]221
[3221a2b]222 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
223 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]224
[3898392]225 //Then publish the results
[13d2dac]226 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
232 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
233 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]234 }
[95fdb0a]235}
236
[29f4fe62]237//===========================================================================================================
238//Routine responsible of sending the email notification once the build is completed
239//===========================================================================================================
[6b6c26e]240@NonCPS
241def SplitLines(String text) {
242 def list = []
243
244 text.eachLine {
245 list += it
246 }
247
248 return list
249}
250
[a336d46]251def GitLogMessage() {
252 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
[e8a22a7]253
[fce01e7]254 def oldRef = Settings.GitOldRef
255 def newRef = Settings.GitNewRef
[6badd87]256
[fdb6ac6]257 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
[6b6c26e]258 def revList = SplitLines( revText )
[6e31c43]259
[fdb6ac6]260 def gitUpdate = ""
261 revList.each { rev ->
[249091f]262 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
[fce01e7]263 gitUpdate = gitUpdate + " via ${rev} (${type})"
264 }
265
266 def rev = oldRef
[249091f]267 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
268 gitUpdate = gitUpdate + " from ${rev} (${type})"
[fce01e7]269
[249091f]270 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
[fce01e7]271
[249091f]272 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
[fce01e7]273 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
274 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
275 gitDiff = gitDiff.replace('[m', '</span>')
[7a927ed0]276
[a336d46]277 return """
[13c98a4]278<pre>
[848fb00]279The branch ${env.BRANCH_NAME} has been updated.
[7a927ed0]280${gitUpdate}
[13c98a4]281</pre>
282
283<p>Check console output at ${env.BUILD_URL} to view the results.</p>
[7b1a604]284
[13c98a4]285<p>- Status --------------------------------------------------------------</p>
[7b1a604]286
[13c98a4]287<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
[7b1a604]288
[13c98a4]289<p>- Log -----------------------------------------------------------------</p>
[e8a22a7]290
[13c98a4]291<pre>
[7a927ed0]292${gitLog}
[13c98a4]293</pre>
294
295<p>-----------------------------------------------------------------------</p>
296<pre>
[7b1a604]297Summary of changes:
[7a927ed0]298${gitDiff}
[13c98a4]299</pre>
[7b1a604]300"""
[a336d46]301}
302
303//Standard build email notification
[13c98a4]304def email(boolean log) {
[a336d46]305 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
306 //Configurations for email format
307 echo 'Notifying users of result'
308
309 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
310 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]311 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]312generated because of a git hooks/post-receive script following
[986e260]313a ref change which was pushed to the C\u2200 repository.</p>
[a336d46]314""" + GitLogMessage()
[e8a22a7]315
[13c98a4]316 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]317
[13c98a4]318 if( Settings && !Settings.Silent ) {
[094a42c]319 //send email notification
320 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
321 } else {
322 echo "Would send email to: ${email_to}"
323 echo "With title: ${email_subject}"
324 echo "Content: \n${email_body}"
325 }
[e8a22a7]326}
[5b8413b4]327
328//===========================================================================================================
329// Helper classes/variables/routines
330//===========================================================================================================
331//Description of a compiler (Must be serializable since pipelines are persistent)
332class CC_Desc implements Serializable {
[93fe3154]333 public String name
334 public String CXX
[6ebc13f]335 public String CC
[bf22bc6]336 public String lto
[93fe3154]337
[bf22bc6]338 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]339 this.name = name
340 this.CXX = CXX
[bf22bc6]341 this.CC = CC
342 this.lto = lto
[5b8413b4]343 }
344}
345
346//Description of an architecture (Must be serializable since pipelines are persistent)
347class Arch_Desc implements Serializable {
348 public String name
349 public String flags
[5307c33]350 public String node
[5b8413b4]351
[5307c33]352 Arch_Desc(String name, String flags, String node) {
[5b8413b4]353 this.name = name
354 this.flags = flags
[5307c33]355 this.node = node
[5b8413b4]356 }
357}
358
359class BuildSettings implements Serializable {
360 public final CC_Desc Compiler
361 public final Arch_Desc Architecture
[d21dd3cb]362 public final Boolean NewAST
[5b8413b4]363 public final Boolean RunAllTests
364 public final Boolean RunBenchmark
365 public final Boolean BuildDocumentation
366 public final Boolean Publish
367 public final Boolean Silent
368 public final Boolean IsSandbox
369 public final String DescLong
370 public final String DescShort
371
[a336d46]372 public String GitNewRef
373 public String GitOldRef
374
[490cb3c]375 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]376 switch( param.Compiler ) {
[099f5bd]377 case 'gcc-9':
[bf22bc6]378 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]379 break
380 case 'gcc-8':
[bf22bc6]381 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]382 break
383 case 'gcc-7':
[bf22bc6]384 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]385 break
[5b8413b4]386 case 'gcc-6':
[bf22bc6]387 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]388 break
389 case 'gcc-5':
[bf22bc6]390 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]391 break
392 case 'gcc-4.9':
[bf22bc6]393 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]394 break
395 case 'clang':
[391c065]396 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]397 break
398 default :
399 error "Unhandled compiler : ${cc}"
400 }
401
402 switch( param.Architecture ) {
403 case 'x64':
[a3e8281]404 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]405 break
406 case 'x86':
[a3e8281]407 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]408 break
409 default :
410 error "Unhandled architecture : ${arch}"
411 }
412
[f95e8f0]413 this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]414 this.NewAST = param.NewAST
[5b8413b4]415 this.RunAllTests = param.RunAllTests
[7a230fd]416 this.RunBenchmark = param.RunBenchmark
[5b8413b4]417 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]418 this.Publish = param.Publish
[5b8413b4]419 this.Silent = param.Silent
420
421 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]422 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]423
[93fe3154]424 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[5b8413b4]425Architecture : ${ this.Architecture.name }
426Arc Flags : ${ this.Architecture.flags }
427Run All Tests : ${ this.RunAllTests.toString() }
428Run Benchmark : ${ this.RunBenchmark.toString() }
429Build Documentation : ${ this.BuildDocumentation.toString() }
430Publish : ${ this.Publish.toString() }
431Silent : ${ this.Silent.toString() }
432"""
[a336d46]433
434 this.GitNewRef = ''
435 this.GitOldRef = ''
[5b8413b4]436 }
437}
438
[490cb3c]439class PlotGroup implements Serializable {
440 public String name
441 public String unit
442 public boolean log
443
444 PlotGroup(String name, String unit, boolean log) {
445 this.name = name
446 this.unit = unit
447 this.log = log
448 }
449}
450
[5b8413b4]451def prepare_build() {
452 // prepare the properties
453 properties ([ \
[62f96ae]454 buildDiscarder(logRotator( \
455 artifactDaysToKeepStr: '', \
456 artifactNumToKeepStr: '', \
457 daysToKeepStr: '730', \
458 numToKeepStr: '1000' \
459 )), \
[5b8413b4]460 [$class: 'ParametersDefinitionProperty', \
461 parameterDefinitions: [ \
462 [$class: 'ChoiceParameterDefinition', \
463 description: 'Which compiler to use', \
464 name: 'Compiler', \
[c09ae73]465 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]466 defaultValue: 'gcc-8', \
[5b8413b4]467 ], \
468 [$class: 'ChoiceParameterDefinition', \
469 description: 'The target architecture', \
470 name: 'Architecture', \
471 choices: 'x64\nx86', \
472 defaultValue: 'x64', \
473 ], \
[d21dd3cb]474 [$class: 'BooleanParameterDefinition', \
475 description: 'If true, build compiler using new AST', \
476 name: 'NewAST', \
477 defaultValue: false, \
478 ], \
[5b8413b4]479 [$class: 'BooleanParameterDefinition', \
480 description: 'If false, only the quick test suite is ran', \
481 name: 'RunAllTests', \
482 defaultValue: false, \
[d21dd3cb]483 ],
[5b8413b4]484 [$class: 'BooleanParameterDefinition', \
485 description: 'If true, jenkins also runs benchmarks', \
486 name: 'RunBenchmark', \
487 defaultValue: false, \
488 ], \
489 [$class: 'BooleanParameterDefinition', \
490 description: 'If true, jenkins also builds documentation', \
491 name: 'BuildDocumentation', \
492 defaultValue: true, \
493 ], \
494 [$class: 'BooleanParameterDefinition', \
495 description: 'If true, jenkins also publishes results', \
496 name: 'Publish', \
497 defaultValue: false, \
498 ], \
499 [$class: 'BooleanParameterDefinition', \
500 description: 'If true, jenkins will not send emails', \
501 name: 'Silent', \
502 defaultValue: false, \
503 ], \
504 ],
505 ]])
506
[4c55047]507 // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer
508 checkout scm
[2407853]509
[490cb3c]510 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]511
512 currentBuild.description = settings.DescShort
513 echo settings.DescLong
514
515 return settings
516}
517
518def make_doc() {
519 def err = null
520 try {
521 sh 'make clean > /dev/null'
522 sh 'make > /dev/null 2>&1'
523 }
524 catch (Exception caughtError) {
525 err = caughtError //rethrow error later
[65f4a51]526 sh 'cat build/*.log'
[5b8413b4]527 }
528 finally {
529 if (err) throw err // Must re-throw exception to propagate error
530 }
[a2a0065]531}
532
[3221a2b]533def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]534
[1b3eef8]535 if(new_data) {
536 echo "Publishing new data"
537 }
538
[cdcd53dc]539 def series = new_data ? [[
[df57a84]540 file: "${file}.csv",
[3c40dc2a]541 exclusionValues: '',
542 displayTableFlag: false,
543 inclusionFlag: 'OFF',
544 url: ''
[cdcd53dc]545 ]] : [];
[8d63649]546
547 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
548 dir("${BuildDir}/benchmark/") {
549 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
550 csvSeries: series,
[490cb3c]551 group: "${group.name}",
[3c40dc2a]552 title: "${title}",
553 style: 'lineSimple',
554 exclZero: false,
555 keepRecords: false,
[3221a2b]556 logarithmic: !relative && group.log,
[3c40dc2a]557 numBuilds: '120',
558 useDescr: true,
[490cb3c]559 yaxis: group.unit,
[3c40dc2a]560 yaxisMaximum: '',
561 yaxisMinimum: ''
[df57a84]562 }
563}
Note: See TracBrowser for help on using the repository browser.