source: Jenkinsfile@ 3489ea6

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 3489ea6 was 9824500, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Fixed email generation in Jenkins
  • Property mode set to 100644
File size: 15.3 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[29f4fe62]5//===========================================================================================================
[9beae23]6// Main loop of the compilation
[29f4fe62]7//===========================================================================================================
[56a9ce6]8
[d3b95f1]9// Globals
[e018546]10BuildDir = null
11SrcDir = null
[d3b95f1]12Settings = null
13Tools = null
[4f9e706]14
[d3b95f1]15// Local variables
16def err = null
17def log_needed = false
[5b8413b4]18
[d3b95f1]19currentBuild.result = "SUCCESS"
[4f9e706]20
[d3b95f1]21try {
[47138ee]22 node {
23 //Wrap build to add timestamp to command line
24 wrap([$class: 'TimestamperBuildWrapper']) {
[85142648]25 Settings = prepare_build()
[d3b95f1]26 }
[47138ee]27 }
[952ee7a]28
[47138ee]29 node(Settings.Architecture.node) {
30 //Wrap build to add timestamp to command line
31 wrap([$class: 'TimestamperBuildWrapper']) {
[d3b95f1]32 BuildDir = pwd tmp: true
33 SrcDir = pwd tmp: false
34 currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
[7aebc62]35
[d3b95f1]36 Tools.Clean()
[fde808df]37
[d3b95f1]38 Tools.Checkout()
[95fdb0a]39
[d3b95f1]40 build()
[95fdb0a]41
[d3b95f1]42 test()
[95fdb0a]43
[d3b95f1]44 benchmark()
[7359098]45
[d3b95f1]46 build_doc()
[9beae23]47
[d3b95f1]48 publish()
[9beae23]49 }
[738cf8f]50 }
[d3b95f1]51}
[738cf8f]52
[d3b95f1]53//If an exception is caught we need to change the status and remember to
54//attach the build log to the email
55catch (Exception caughtError) {
56 // Store the result of the build log
57 currentBuild.result = "FAILURE"
[9dd31e7]58
[d3b95f1]59 // An error has occured, the build log is relevent
60 log_needed = true
[9beae23]61
[d3b95f1]62 // rethrow error later
63 err = caughtError
[852ae0ea]64
[d3b95f1]65 // print the error so it shows in the log
66 echo err.toString()
67}
[738cf8f]68
[d3b95f1]69finally {
70 //Send email with final results if this is not a full build
71 email(log_needed)
[9beae23]72
[d3b95f1]73 echo 'Build Completed'
[734891d]74
[d3b95f1]75 /* Must re-throw exception to propagate error */
76 if (err) {
77 throw err
[738cf8f]78 }
79}
[29f4fe62]80//===========================================================================================================
[9beae23]81// Main compilation routines
[29f4fe62]82//===========================================================================================================
[95fdb0a]83def build() {
[e507c11]84 debug = true
85 release = Settings.RunAllTests || Settings.RunBenchmark
[91aa5ab]86 Tools.BuildStage('Build : configure', true) {
[dcf1979]87 // Configure must be run inside the tree
88 dir (SrcDir) {
89 // Generate the necessary build files
90 sh './autogen.sh'
91 }
92
[50f2cfc]93 // Build outside of the src tree to ease cleaning
[5b8413b4]94 dir (BuildDir) {
[8e58264]95 //Configure the compilation (Output is not relevant)
[50f2cfc]96 //Use the current directory as the installation target so nothing escapes the sandbox
97 //Also specify the compiler by hand
[3fc5f010]98 targets=""
[d4510ea]99 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]100 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]101 } else {
[6bde81d]102 targets="--with-target-hosts='host:debug'"
[3fc5f010]103 }
104
[d21dd3cb]105 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
106
107 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]108
109 // Configure libcfa
[e70e54e]110 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]111 }
112 }
113
[91aa5ab]114 Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]115 // Build outside of the src tree to ease cleaning
116 dir (BuildDir) {
117 // Build driver
118 sh 'make -j 8 --no-print-directory -C driver'
119
120 // Build translator
121 sh 'make -j 8 --no-print-directory -C src'
122 }
123 }
[1752d0e]124
[91aa5ab]125 Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]126 // Build outside of the src tree to ease cleaning
127 dir (BuildDir) {
128 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
129 }
130 }
131
[91aa5ab]132 Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]133 // Build outside of the src tree to ease cleaning
134 dir (BuildDir) {
135 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]136 }
[620dd2b]137 }
[aa96fba]138
[91aa5ab]139 Tools.BuildStage('Build : install', true) {
[aa96fba]140 // Build outside of the src tree to ease cleaning
141 dir (BuildDir) {
142 sh "make -j 8 --no-print-directory install"
143 }
144 }
[95fdb0a]145}
[24eecab]146
[95fdb0a]147def test() {
[4c1b9ea8]148 try {
[1baf6ed]149 // Print potential limits before testing
150 // in case jenkins messes with them
[a33dcd5]151 sh 'free -h'
[1baf6ed]152 sh 'ulimit -a'
153
[91aa5ab]154 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]155 dir (BuildDir) {
[3e93c00]156 //Run the tests from the tests directory
[4c51aca]157 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]158 }
[4c1b9ea8]159 }
160
[91aa5ab]161 Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]162 dir (BuildDir) {
163 //Run the tests from the tests directory
[4c51aca]164 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
165 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]166 }
[9beae23]167 }
[620dd2b]168 }
[4c1b9ea8]169 catch (Exception err) {
170 echo "Archiving core dumps"
[c95fdc9]171 dir (BuildDir) {
[cc9ec56]172 def exists = fileExists 'tests/crashes'
173 if( exists ) {
[cece53c]174 sh """${SrcDir}/tools/jenkins/archive-gen.sh"""
175 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*,setup.sh", fingerprint: true
176 }
[c95fdc9]177 }
[4c1b9ea8]178 throw err
179 }
[95fdb0a]180}
181
182def benchmark() {
[91aa5ab]183 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]184 dir (BuildDir) {
[0dc3ac3]185 //Append bench results
[c6f1f3e]186 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]187 }
[620dd2b]188 }
[9beae23]189}
[efd60d67]190
[95fdb0a]191def build_doc() {
[91aa5ab]192 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]193 dir ('doc/user') {
194 make_doc()
195 }
196
197 dir ('doc/refrat') {
198 make_doc()
199 }
[620dd2b]200 }
[9beae23]201}
202
[95fdb0a]203def publish() {
[91aa5ab]204 Tools.BuildStage('Publish', true) {
[95fdb0a]205
[1b3eef8]206 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]207
[3221a2b]208 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
209 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]210
[3898392]211 //Then publish the results
[13d2dac]212 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
213 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
214 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
215 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
216 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
217 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
218 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
219 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]220 }
[95fdb0a]221}
222
[29f4fe62]223//===========================================================================================================
224//Routine responsible of sending the email notification once the build is completed
225//===========================================================================================================
[a336d46]226//Standard build email notification
[13c98a4]227def email(boolean log) {
[bd50205]228 node {
229 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
230 //Configurations for email format
231 echo 'Notifying users of result'
232
233 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
234 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
235 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[9824500]236 generated because of a git hooks/post-receive script following
237 a ref change which was pushed to the C\u2200 repository.</p>
238
239 <p>- Status --------------------------------------------------------------</p>
240
241 <p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
242 <p>Check console output at ${env.BUILD_URL} to view the results.</p>
243 """ + Tools.GitLogMessage()
[bd50205]244
245 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
246
247 if( Settings && !Settings.Silent ) {
248 //send email notification
249 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
250 } else {
251 echo "Would send email to: ${email_to}"
252 echo "With title: ${email_subject}"
253 echo "Content: \n${email_body}"
254 }
[094a42c]255 }
[e8a22a7]256}
[5b8413b4]257
258//===========================================================================================================
259// Helper classes/variables/routines
260//===========================================================================================================
261//Description of a compiler (Must be serializable since pipelines are persistent)
262class CC_Desc implements Serializable {
[93fe3154]263 public String name
264 public String CXX
[6ebc13f]265 public String CC
[bf22bc6]266 public String lto
[93fe3154]267
[bf22bc6]268 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]269 this.name = name
270 this.CXX = CXX
[bf22bc6]271 this.CC = CC
272 this.lto = lto
[5b8413b4]273 }
274}
275
276//Description of an architecture (Must be serializable since pipelines are persistent)
277class Arch_Desc implements Serializable {
278 public String name
279 public String flags
[5307c33]280 public String node
[5b8413b4]281
[5307c33]282 Arch_Desc(String name, String flags, String node) {
[5b8413b4]283 this.name = name
284 this.flags = flags
[5307c33]285 this.node = node
[5b8413b4]286 }
287}
288
289class BuildSettings implements Serializable {
290 public final CC_Desc Compiler
291 public final Arch_Desc Architecture
[d21dd3cb]292 public final Boolean NewAST
[5b8413b4]293 public final Boolean RunAllTests
294 public final Boolean RunBenchmark
295 public final Boolean BuildDocumentation
296 public final Boolean Publish
297 public final Boolean Silent
298 public final Boolean IsSandbox
299 public final String DescLong
300 public final String DescShort
301
[a336d46]302 public String GitNewRef
303 public String GitOldRef
304
[490cb3c]305 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]306 switch( param.Compiler ) {
[099f5bd]307 case 'gcc-9':
[bf22bc6]308 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]309 break
310 case 'gcc-8':
[bf22bc6]311 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]312 break
313 case 'gcc-7':
[bf22bc6]314 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]315 break
[5b8413b4]316 case 'gcc-6':
[bf22bc6]317 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]318 break
319 case 'gcc-5':
[bf22bc6]320 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]321 break
322 case 'gcc-4.9':
[bf22bc6]323 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]324 break
325 case 'clang':
[391c065]326 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]327 break
328 default :
329 error "Unhandled compiler : ${cc}"
330 }
331
332 switch( param.Architecture ) {
333 case 'x64':
[a3e8281]334 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]335 break
336 case 'x86':
[a3e8281]337 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]338 break
339 default :
340 error "Unhandled architecture : ${arch}"
341 }
342
[f95e8f0]343 this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]344 this.NewAST = param.NewAST
[5b8413b4]345 this.RunAllTests = param.RunAllTests
[7a230fd]346 this.RunBenchmark = param.RunBenchmark
[5b8413b4]347 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]348 this.Publish = param.Publish
[5b8413b4]349 this.Silent = param.Silent
350
351 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]352 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]353
[fe3d9ab]354 final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]355 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[85799aa]356AST Version : ${ ast.toString() }
[5b8413b4]357Architecture : ${ this.Architecture.name }
358Arc Flags : ${ this.Architecture.flags }
359Run All Tests : ${ this.RunAllTests.toString() }
360Run Benchmark : ${ this.RunBenchmark.toString() }
361Build Documentation : ${ this.BuildDocumentation.toString() }
362Publish : ${ this.Publish.toString() }
363Silent : ${ this.Silent.toString() }
364"""
[a336d46]365
366 this.GitNewRef = ''
367 this.GitOldRef = ''
[5b8413b4]368 }
369}
370
[490cb3c]371class PlotGroup implements Serializable {
372 public String name
373 public String unit
374 public boolean log
375
376 PlotGroup(String name, String unit, boolean log) {
377 this.name = name
378 this.unit = unit
379 this.log = log
380 }
381}
382
[5b8413b4]383def prepare_build() {
384 // prepare the properties
385 properties ([ \
[62f96ae]386 buildDiscarder(logRotator( \
387 artifactDaysToKeepStr: '', \
388 artifactNumToKeepStr: '', \
389 daysToKeepStr: '730', \
390 numToKeepStr: '1000' \
391 )), \
[5b8413b4]392 [$class: 'ParametersDefinitionProperty', \
393 parameterDefinitions: [ \
394 [$class: 'ChoiceParameterDefinition', \
395 description: 'Which compiler to use', \
396 name: 'Compiler', \
[c09ae73]397 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]398 defaultValue: 'gcc-8', \
[5b8413b4]399 ], \
400 [$class: 'ChoiceParameterDefinition', \
401 description: 'The target architecture', \
402 name: 'Architecture', \
403 choices: 'x64\nx86', \
404 defaultValue: 'x64', \
405 ], \
[d21dd3cb]406 [$class: 'BooleanParameterDefinition', \
407 description: 'If true, build compiler using new AST', \
408 name: 'NewAST', \
[6a531ab]409 defaultValue: true, \
[d21dd3cb]410 ], \
[5b8413b4]411 [$class: 'BooleanParameterDefinition', \
412 description: 'If false, only the quick test suite is ran', \
413 name: 'RunAllTests', \
414 defaultValue: false, \
[fe3d9ab]415 ], \
[5b8413b4]416 [$class: 'BooleanParameterDefinition', \
417 description: 'If true, jenkins also runs benchmarks', \
418 name: 'RunBenchmark', \
419 defaultValue: false, \
420 ], \
421 [$class: 'BooleanParameterDefinition', \
422 description: 'If true, jenkins also builds documentation', \
423 name: 'BuildDocumentation', \
424 defaultValue: true, \
425 ], \
426 [$class: 'BooleanParameterDefinition', \
427 description: 'If true, jenkins also publishes results', \
428 name: 'Publish', \
429 defaultValue: false, \
430 ], \
431 [$class: 'BooleanParameterDefinition', \
432 description: 'If true, jenkins will not send emails', \
433 name: 'Silent', \
434 defaultValue: false, \
435 ], \
436 ],
437 ]])
438
[bd8dca2]439 // It's unfortunate but it looks like we need to checkout the entire repo just to get
440 // - the pretty git printer
441 // - Jenkins.tools
[4c55047]442 checkout scm
[2407853]443
[1483a16]444 Tools = load "Jenkins/tools.groovy"
[bd8dca2]445
[490cb3c]446 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]447
448 currentBuild.description = settings.DescShort
449 echo settings.DescLong
450
451 return settings
452}
453
454def make_doc() {
455 def err = null
456 try {
457 sh 'make clean > /dev/null'
458 sh 'make > /dev/null 2>&1'
459 }
460 catch (Exception caughtError) {
461 err = caughtError //rethrow error later
[65f4a51]462 sh 'cat build/*.log'
[5b8413b4]463 }
464 finally {
465 if (err) throw err // Must re-throw exception to propagate error
466 }
[a2a0065]467}
468
[3221a2b]469def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]470
[1b3eef8]471 if(new_data) {
472 echo "Publishing new data"
473 }
474
[cdcd53dc]475 def series = new_data ? [[
[df57a84]476 file: "${file}.csv",
[3c40dc2a]477 exclusionValues: '',
478 displayTableFlag: false,
479 inclusionFlag: 'OFF',
480 url: ''
[cdcd53dc]481 ]] : [];
[8d63649]482
483 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
484 dir("${BuildDir}/benchmark/") {
485 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
486 csvSeries: series,
[490cb3c]487 group: "${group.name}",
[3c40dc2a]488 title: "${title}",
489 style: 'lineSimple',
490 exclZero: false,
491 keepRecords: false,
[3221a2b]492 logarithmic: !relative && group.log,
[3c40dc2a]493 numBuilds: '120',
494 useDescr: true,
[490cb3c]495 yaxis: group.unit,
[3c40dc2a]496 yaxisMaximum: '',
497 yaxisMinimum: ''
[df57a84]498 }
499}
Note: See TracBrowser for help on using the repository browser.