source: Jenkinsfile@ 8ca82de

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 8ca82de was 8ca82de, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

More attempts at loading tools
  • Property mode set to 100644
File size: 16.8 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[29f4fe62]5//===========================================================================================================
[9beae23]6// Main loop of the compilation
[29f4fe62]7//===========================================================================================================
[56a9ce6]8
[85142648]9node('master') {
10 // Globals
11 BuildDir = pwd tmp: true
12 SrcDir = pwd tmp: false
13 Settings = null
[4f9e706]14
[85142648]15 // Local variables
16 def err = null
17 def log_needed = false
[5b8413b4]18
[85142648]19 currentBuild.result = "SUCCESS"
[4f9e706]20
[85142648]21 try {
22 //Wrap build to add timestamp to command line
23 wrap([$class: 'TimestamperBuildWrapper']) {
[0ef06b6]24
[85142648]25 Settings = prepare_build()
[952ee7a]26
[85142648]27 node(Settings.Architecture.node) {
28 BuildDir = pwd tmp: true
29 SrcDir = pwd tmp: false
[23a14d86]30
[85142648]31 clean()
[7aebc62]32
[85142648]33 checkout()
[fde808df]34
[85142648]35 build()
[95fdb0a]36
[85142648]37 test()
[95fdb0a]38
[85142648]39 benchmark()
[95fdb0a]40
[85142648]41 build_doc()
[7359098]42
[85142648]43 publish()
44 }
[9beae23]45
[85142648]46 // Update the build directories when exiting the node
47 BuildDir = pwd tmp: true
48 SrcDir = pwd tmp: false
[9beae23]49 }
[738cf8f]50 }
51
[85142648]52 //If an exception is caught we need to change the status and remember to
53 //attach the build log to the email
54 catch (Exception caughtError) {
55 //rethrow error later
56 err = caughtError
[738cf8f]57
[85142648]58 echo err.toString()
[e966ec0]59
[85142648]60 //An error has occured, the build log is relevent
61 log_needed = true
[9beae23]62
[85142648]63 //Store the result of the build log
[27b1ca1]64 currentBuild.result = "${tools.StageName} FAILURE".trim()
[85142648]65 }
[738cf8f]66
[85142648]67 finally {
68 //Send email with final results if this is not a full build
69 email(log_needed)
[9beae23]70
[85142648]71 echo 'Build Completed'
[734891d]72
[85142648]73 /* Must re-throw exception to propagate error */
74 if (err) {
75 throw err
76 }
[738cf8f]77 }
78}
[29f4fe62]79//===========================================================================================================
[9beae23]80// Main compilation routines
[29f4fe62]81//===========================================================================================================
[0dc3ac3]82def clean() {
[6c55a3d]83 build_stage('Cleanup', true) {
[0dc3ac3]84 // clean the build by wipping the build directory
[5b8413b4]85 dir(BuildDir) {
[d4cd491]86 deleteDir()
[ece8a80]87 }
[620dd2b]88 }
[95fdb0a]89}
[29f4fe62]90
[0dc3ac3]91//Compilation script is done here but environnement set-up and error handling is done in main loop
92def checkout() {
[6c55a3d]93 build_stage('Checkout', true) {
[0dc3ac3]94 //checkout the source code and clean the repo
[a336d46]95 final scmVars = checkout scm
96 Settings.GitNewRef = scmVars.GIT_COMMIT
97 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
[d3a4564a]98
[a336d46]99 echo GitLogMessage()
[0dc3ac3]100 }
101}
102
[95fdb0a]103def build() {
[e507c11]104 debug = true
105 release = Settings.RunAllTests || Settings.RunBenchmark
106 build_stage('Build : configure', true) {
[dcf1979]107 // Configure must be run inside the tree
108 dir (SrcDir) {
109 // Generate the necessary build files
110 sh './autogen.sh'
111 }
112
[50f2cfc]113 // Build outside of the src tree to ease cleaning
[5b8413b4]114 dir (BuildDir) {
[50f2cfc]115 //Configure the conpilation (Output is not relevant)
116 //Use the current directory as the installation target so nothing escapes the sandbox
117 //Also specify the compiler by hand
[3fc5f010]118 targets=""
[d4510ea]119 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]120 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]121 } else {
[6bde81d]122 targets="--with-target-hosts='host:debug'"
[3fc5f010]123 }
124
[d21dd3cb]125 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
126
127 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]128
129 // Configure libcfa
[e70e54e]130 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]131 }
132 }
133
134 build_stage('Build : cfa-cpp', true) {
135 // Build outside of the src tree to ease cleaning
136 dir (BuildDir) {
137 // Build driver
138 sh 'make -j 8 --no-print-directory -C driver'
139
140 // Build translator
141 sh 'make -j 8 --no-print-directory -C src'
142 }
143 }
[1752d0e]144
[e507c11]145 build_stage('Build : libcfa(debug)', debug) {
146 // Build outside of the src tree to ease cleaning
147 dir (BuildDir) {
148 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
149 }
150 }
151
152 build_stage('Build : libcfa(nodebug)', release) {
153 // Build outside of the src tree to ease cleaning
154 dir (BuildDir) {
155 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]156 }
[620dd2b]157 }
[aa96fba]158
159 build_stage('Build : install', true) {
160 // Build outside of the src tree to ease cleaning
161 dir (BuildDir) {
162 sh "make -j 8 --no-print-directory install"
163 }
164 }
[95fdb0a]165}
[24eecab]166
[95fdb0a]167def test() {
[4c1b9ea8]168 try {
169 build_stage('Test: short', !Settings.RunAllTests) {
170 dir (BuildDir) {
[3e93c00]171 //Run the tests from the tests directory
[4c51aca]172 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]173 }
[4c1b9ea8]174 }
175
176 build_stage('Test: full', Settings.RunAllTests) {
177 dir (BuildDir) {
178 //Run the tests from the tests directory
[4c51aca]179 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
180 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]181 }
[9beae23]182 }
[620dd2b]183 }
[4c1b9ea8]184 catch (Exception err) {
185 echo "Archiving core dumps"
[c95fdc9]186 dir (BuildDir) {
[62f96ae]187 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
[c95fdc9]188 }
[4c1b9ea8]189 throw err
190 }
[95fdb0a]191}
192
193def benchmark() {
[6c55a3d]194 build_stage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]195 dir (BuildDir) {
[0dc3ac3]196 //Append bench results
[c6f1f3e]197 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]198 }
[620dd2b]199 }
[9beae23]200}
[efd60d67]201
[95fdb0a]202def build_doc() {
[6c55a3d]203 build_stage('Documentation', Settings.BuildDocumentation) {
[9beae23]204 dir ('doc/user') {
205 make_doc()
206 }
207
208 dir ('doc/refrat') {
209 make_doc()
210 }
[620dd2b]211 }
[9beae23]212}
213
[95fdb0a]214def publish() {
[6c55a3d]215 build_stage('Publish', true) {
[95fdb0a]216
[1b3eef8]217 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]218
[3221a2b]219 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
220 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]221
[3898392]222 //Then publish the results
[13d2dac]223 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
224 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
225 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
226 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]231 }
[95fdb0a]232}
233
[29f4fe62]234//===========================================================================================================
235//Routine responsible of sending the email notification once the build is completed
236//===========================================================================================================
[6b6c26e]237@NonCPS
238def SplitLines(String text) {
239 def list = []
240
241 text.eachLine {
242 list += it
243 }
244
245 return list
246}
247
[a336d46]248def GitLogMessage() {
249 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
[e8a22a7]250
[fce01e7]251 def oldRef = Settings.GitOldRef
252 def newRef = Settings.GitNewRef
[6badd87]253
[fdb6ac6]254 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
[6b6c26e]255 def revList = SplitLines( revText )
[6e31c43]256
[fdb6ac6]257 def gitUpdate = ""
258 revList.each { rev ->
[249091f]259 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
[fce01e7]260 gitUpdate = gitUpdate + " via ${rev} (${type})"
261 }
262
263 def rev = oldRef
[249091f]264 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
265 gitUpdate = gitUpdate + " from ${rev} (${type})"
[fce01e7]266
[249091f]267 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
[fce01e7]268
[249091f]269 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
[fce01e7]270 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
271 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
272 gitDiff = gitDiff.replace('[m', '</span>')
[7a927ed0]273
[a336d46]274 return """
[13c98a4]275<pre>
[848fb00]276The branch ${env.BRANCH_NAME} has been updated.
[7a927ed0]277${gitUpdate}
[13c98a4]278</pre>
279
280<p>Check console output at ${env.BUILD_URL} to view the results.</p>
[7b1a604]281
[13c98a4]282<p>- Status --------------------------------------------------------------</p>
[7b1a604]283
[13c98a4]284<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
[7b1a604]285
[13c98a4]286<p>- Log -----------------------------------------------------------------</p>
[e8a22a7]287
[13c98a4]288<pre>
[7a927ed0]289${gitLog}
[13c98a4]290</pre>
291
292<p>-----------------------------------------------------------------------</p>
293<pre>
[7b1a604]294Summary of changes:
[7a927ed0]295${gitDiff}
[13c98a4]296</pre>
[7b1a604]297"""
[a336d46]298}
299
300//Standard build email notification
[13c98a4]301def email(boolean log) {
[a336d46]302 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
303 //Configurations for email format
304 echo 'Notifying users of result'
305
306 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
307 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]308 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]309generated because of a git hooks/post-receive script following
[986e260]310a ref change which was pushed to the C\u2200 repository.</p>
[a336d46]311""" + GitLogMessage()
[e8a22a7]312
[13c98a4]313 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]314
[13c98a4]315 if( Settings && !Settings.Silent ) {
[094a42c]316 //send email notification
317 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
318 } else {
319 echo "Would send email to: ${email_to}"
320 echo "With title: ${email_subject}"
321 echo "Content: \n${email_body}"
322 }
[e8a22a7]323}
[5b8413b4]324
325//===========================================================================================================
326// Helper classes/variables/routines
327//===========================================================================================================
328//Description of a compiler (Must be serializable since pipelines are persistent)
329class CC_Desc implements Serializable {
[93fe3154]330 public String name
331 public String CXX
[6ebc13f]332 public String CC
[bf22bc6]333 public String lto
[93fe3154]334
[bf22bc6]335 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]336 this.name = name
337 this.CXX = CXX
[bf22bc6]338 this.CC = CC
339 this.lto = lto
[5b8413b4]340 }
341}
342
343//Description of an architecture (Must be serializable since pipelines are persistent)
344class Arch_Desc implements Serializable {
345 public String name
346 public String flags
[5307c33]347 public String node
[5b8413b4]348
[5307c33]349 Arch_Desc(String name, String flags, String node) {
[5b8413b4]350 this.name = name
351 this.flags = flags
[5307c33]352 this.node = node
[5b8413b4]353 }
354}
355
356class BuildSettings implements Serializable {
357 public final CC_Desc Compiler
358 public final Arch_Desc Architecture
[d21dd3cb]359 public final Boolean NewAST
[5b8413b4]360 public final Boolean RunAllTests
361 public final Boolean RunBenchmark
362 public final Boolean BuildDocumentation
363 public final Boolean Publish
364 public final Boolean Silent
365 public final Boolean IsSandbox
366 public final String DescLong
367 public final String DescShort
368
[a336d46]369 public String GitNewRef
370 public String GitOldRef
371
[490cb3c]372 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]373 switch( param.Compiler ) {
[099f5bd]374 case 'gcc-9':
[bf22bc6]375 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]376 break
377 case 'gcc-8':
[bf22bc6]378 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]379 break
380 case 'gcc-7':
[bf22bc6]381 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]382 break
[5b8413b4]383 case 'gcc-6':
[bf22bc6]384 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]385 break
386 case 'gcc-5':
[bf22bc6]387 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]388 break
389 case 'gcc-4.9':
[bf22bc6]390 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]391 break
392 case 'clang':
[391c065]393 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]394 break
395 default :
396 error "Unhandled compiler : ${cc}"
397 }
398
399 switch( param.Architecture ) {
400 case 'x64':
[a3e8281]401 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]402 break
403 case 'x86':
[a3e8281]404 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]405 break
406 default :
407 error "Unhandled architecture : ${arch}"
408 }
409
[f95e8f0]410 this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]411 this.NewAST = param.NewAST
[5b8413b4]412 this.RunAllTests = param.RunAllTests
[7a230fd]413 this.RunBenchmark = param.RunBenchmark
[5b8413b4]414 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]415 this.Publish = param.Publish
[5b8413b4]416 this.Silent = param.Silent
417
418 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]419 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]420
[93fe3154]421 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[5b8413b4]422Architecture : ${ this.Architecture.name }
423Arc Flags : ${ this.Architecture.flags }
424Run All Tests : ${ this.RunAllTests.toString() }
425Run Benchmark : ${ this.RunBenchmark.toString() }
426Build Documentation : ${ this.BuildDocumentation.toString() }
427Publish : ${ this.Publish.toString() }
428Silent : ${ this.Silent.toString() }
429"""
[a336d46]430
431 this.GitNewRef = ''
432 this.GitOldRef = ''
[5b8413b4]433 }
434}
435
[490cb3c]436class PlotGroup implements Serializable {
437 public String name
438 public String unit
439 public boolean log
440
441 PlotGroup(String name, String unit, boolean log) {
442 this.name = name
443 this.unit = unit
444 this.log = log
445 }
446}
447
[5b8413b4]448def prepare_build() {
449 // prepare the properties
450 properties ([ \
[62f96ae]451 buildDiscarder(logRotator( \
452 artifactDaysToKeepStr: '', \
453 artifactNumToKeepStr: '', \
454 daysToKeepStr: '730', \
455 numToKeepStr: '1000' \
456 )), \
[5b8413b4]457 [$class: 'ParametersDefinitionProperty', \
458 parameterDefinitions: [ \
459 [$class: 'ChoiceParameterDefinition', \
460 description: 'Which compiler to use', \
461 name: 'Compiler', \
[c09ae73]462 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]463 defaultValue: 'gcc-8', \
[5b8413b4]464 ], \
465 [$class: 'ChoiceParameterDefinition', \
466 description: 'The target architecture', \
467 name: 'Architecture', \
468 choices: 'x64\nx86', \
469 defaultValue: 'x64', \
470 ], \
[d21dd3cb]471 [$class: 'BooleanParameterDefinition', \
472 description: 'If true, build compiler using new AST', \
473 name: 'NewAST', \
474 defaultValue: false, \
475 ], \
[5b8413b4]476 [$class: 'BooleanParameterDefinition', \
477 description: 'If false, only the quick test suite is ran', \
478 name: 'RunAllTests', \
479 defaultValue: false, \
[d21dd3cb]480 ],
[5b8413b4]481 [$class: 'BooleanParameterDefinition', \
482 description: 'If true, jenkins also runs benchmarks', \
483 name: 'RunBenchmark', \
484 defaultValue: false, \
485 ], \
486 [$class: 'BooleanParameterDefinition', \
487 description: 'If true, jenkins also builds documentation', \
488 name: 'BuildDocumentation', \
489 defaultValue: true, \
490 ], \
491 [$class: 'BooleanParameterDefinition', \
492 description: 'If true, jenkins also publishes results', \
493 name: 'Publish', \
494 defaultValue: false, \
495 ], \
496 [$class: 'BooleanParameterDefinition', \
497 description: 'If true, jenkins will not send emails', \
498 name: 'Silent', \
499 defaultValue: false, \
500 ], \
501 ],
502 ]])
503
[bd8dca2]504 // It's unfortunate but it looks like we need to checkout the entire repo just to get
505 // - the pretty git printer
506 // - Jenkins.tools
[4c55047]507 checkout scm
[2407853]508
[8ca82de]509 tools = load "Jenkins/tools.groovy"
[d8d9130f]510 build_stage = tools.build_stage
[bd8dca2]511
[490cb3c]512 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]513
514 currentBuild.description = settings.DescShort
515 echo settings.DescLong
516
517 return settings
518}
519
520def make_doc() {
521 def err = null
522 try {
523 sh 'make clean > /dev/null'
524 sh 'make > /dev/null 2>&1'
525 }
526 catch (Exception caughtError) {
527 err = caughtError //rethrow error later
[65f4a51]528 sh 'cat build/*.log'
[5b8413b4]529 }
530 finally {
531 if (err) throw err // Must re-throw exception to propagate error
532 }
[a2a0065]533}
534
[3221a2b]535def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]536
[1b3eef8]537 if(new_data) {
538 echo "Publishing new data"
539 }
540
[cdcd53dc]541 def series = new_data ? [[
[df57a84]542 file: "${file}.csv",
[3c40dc2a]543 exclusionValues: '',
544 displayTableFlag: false,
545 inclusionFlag: 'OFF',
546 url: ''
[cdcd53dc]547 ]] : [];
[8d63649]548
549 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
550 dir("${BuildDir}/benchmark/") {
551 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
552 csvSeries: series,
[490cb3c]553 group: "${group.name}",
[3c40dc2a]554 title: "${title}",
555 style: 'lineSimple',
556 exclZero: false,
557 keepRecords: false,
[3221a2b]558 logarithmic: !relative && group.log,
[3c40dc2a]559 numBuilds: '120',
560 useDescr: true,
[490cb3c]561 yaxis: group.unit,
[3c40dc2a]562 yaxisMaximum: '',
563 yaxisMinimum: ''
[df57a84]564 }
565}
Note: See TracBrowser for help on using the repository browser.