source: Jenkinsfile@ b399ca2

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since b399ca2 was b399ca2, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Testing some scripting options
  • Property mode set to 100644
File size: 14.9 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[29f4fe62]5//===========================================================================================================
[9beae23]6// Main loop of the compilation
[29f4fe62]7//===========================================================================================================
[56a9ce6]8
[85142648]9node('master') {
[b399ca2]10 println scm.properties.sort{it.key}.collect{it}.findAll{!['class', 'active'].contains(it.key)}.join('\n')
11
[85142648]12 // Globals
13 BuildDir = pwd tmp: true
14 SrcDir = pwd tmp: false
15 Settings = null
[1483a16]16 Tools = null
[4f9e706]17
[85142648]18 // Local variables
19 def err = null
20 def log_needed = false
[5b8413b4]21
[85142648]22 currentBuild.result = "SUCCESS"
[4f9e706]23
[85142648]24 try {
25 //Wrap build to add timestamp to command line
26 wrap([$class: 'TimestamperBuildWrapper']) {
[0ef06b6]27
[85142648]28 Settings = prepare_build()
[952ee7a]29
[85142648]30 node(Settings.Architecture.node) {
31 BuildDir = pwd tmp: true
32 SrcDir = pwd tmp: false
[23a14d86]33
[6a531ab]34 Tools.Clean()
[7aebc62]35
[6a531ab]36 Tools.Checkout()
[fde808df]37
[85142648]38 build()
[95fdb0a]39
[85142648]40 test()
[95fdb0a]41
[85142648]42 benchmark()
[95fdb0a]43
[85142648]44 build_doc()
[7359098]45
[85142648]46 publish()
47 }
[9beae23]48
[85142648]49 // Update the build directories when exiting the node
50 BuildDir = pwd tmp: true
51 SrcDir = pwd tmp: false
[9beae23]52 }
[738cf8f]53 }
54
[85142648]55 //If an exception is caught we need to change the status and remember to
56 //attach the build log to the email
57 catch (Exception caughtError) {
58 //rethrow error later
59 err = caughtError
[738cf8f]60
[85142648]61 echo err.toString()
[e966ec0]62
[85142648]63 //An error has occured, the build log is relevent
64 log_needed = true
[9beae23]65
[85142648]66 //Store the result of the build log
[27b1ca1]67 currentBuild.result = "${tools.StageName} FAILURE".trim()
[85142648]68 }
[738cf8f]69
[85142648]70 finally {
71 //Send email with final results if this is not a full build
72 email(log_needed)
[9beae23]73
[85142648]74 echo 'Build Completed'
[734891d]75
[85142648]76 /* Must re-throw exception to propagate error */
77 if (err) {
78 throw err
79 }
[738cf8f]80 }
81}
[29f4fe62]82//===========================================================================================================
[9beae23]83// Main compilation routines
[29f4fe62]84//===========================================================================================================
[95fdb0a]85def build() {
[e507c11]86 debug = true
87 release = Settings.RunAllTests || Settings.RunBenchmark
[91aa5ab]88 Tools.BuildStage('Build : configure', true) {
[dcf1979]89 // Configure must be run inside the tree
90 dir (SrcDir) {
91 // Generate the necessary build files
92 sh './autogen.sh'
93 }
94
[50f2cfc]95 // Build outside of the src tree to ease cleaning
[5b8413b4]96 dir (BuildDir) {
[8e58264]97 //Configure the compilation (Output is not relevant)
[50f2cfc]98 //Use the current directory as the installation target so nothing escapes the sandbox
99 //Also specify the compiler by hand
[3fc5f010]100 targets=""
[d4510ea]101 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]102 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]103 } else {
[6bde81d]104 targets="--with-target-hosts='host:debug'"
[3fc5f010]105 }
106
[d21dd3cb]107 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
108
109 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]110
111 // Configure libcfa
[e70e54e]112 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]113 }
114 }
115
[91aa5ab]116 Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]117 // Build outside of the src tree to ease cleaning
118 dir (BuildDir) {
119 // Build driver
120 sh 'make -j 8 --no-print-directory -C driver'
121
122 // Build translator
123 sh 'make -j 8 --no-print-directory -C src'
124 }
125 }
[1752d0e]126
[91aa5ab]127 Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]128 // Build outside of the src tree to ease cleaning
129 dir (BuildDir) {
130 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
131 }
132 }
133
[91aa5ab]134 Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]135 // Build outside of the src tree to ease cleaning
136 dir (BuildDir) {
137 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]138 }
[620dd2b]139 }
[aa96fba]140
[91aa5ab]141 Tools.BuildStage('Build : install', true) {
[aa96fba]142 // Build outside of the src tree to ease cleaning
143 dir (BuildDir) {
144 sh "make -j 8 --no-print-directory install"
145 }
146 }
[95fdb0a]147}
[24eecab]148
[95fdb0a]149def test() {
[4c1b9ea8]150 try {
[91aa5ab]151 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]152 dir (BuildDir) {
[3e93c00]153 //Run the tests from the tests directory
[4c51aca]154 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]155 }
[4c1b9ea8]156 }
157
[91aa5ab]158 Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]159 dir (BuildDir) {
160 //Run the tests from the tests directory
[4c51aca]161 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
162 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]163 }
[9beae23]164 }
[620dd2b]165 }
[4c1b9ea8]166 catch (Exception err) {
167 echo "Archiving core dumps"
[c95fdc9]168 dir (BuildDir) {
[62f96ae]169 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
[c95fdc9]170 }
[4c1b9ea8]171 throw err
172 }
[95fdb0a]173}
174
175def benchmark() {
[91aa5ab]176 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]177 dir (BuildDir) {
[0dc3ac3]178 //Append bench results
[c6f1f3e]179 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]180 }
[620dd2b]181 }
[9beae23]182}
[efd60d67]183
[95fdb0a]184def build_doc() {
[91aa5ab]185 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]186 dir ('doc/user') {
187 make_doc()
188 }
189
190 dir ('doc/refrat') {
191 make_doc()
192 }
[620dd2b]193 }
[9beae23]194}
195
[95fdb0a]196def publish() {
[91aa5ab]197 Tools.BuildStage('Publish', true) {
[95fdb0a]198
[1b3eef8]199 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]200
[3221a2b]201 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
202 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]203
[3898392]204 //Then publish the results
[13d2dac]205 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
206 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
207 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
208 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
209 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
210 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
211 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
212 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]213 }
[95fdb0a]214}
215
[29f4fe62]216//===========================================================================================================
217//Routine responsible of sending the email notification once the build is completed
218//===========================================================================================================
[a336d46]219//Standard build email notification
[13c98a4]220def email(boolean log) {
[a336d46]221 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
222 //Configurations for email format
223 echo 'Notifying users of result'
224
225 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
226 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]227 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]228generated because of a git hooks/post-receive script following
[986e260]229a ref change which was pushed to the C\u2200 repository.</p>
[6a531ab]230""" + Tools.GitLogMessage()
[e8a22a7]231
[13c98a4]232 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]233
[13c98a4]234 if( Settings && !Settings.Silent ) {
[094a42c]235 //send email notification
236 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
237 } else {
238 echo "Would send email to: ${email_to}"
239 echo "With title: ${email_subject}"
240 echo "Content: \n${email_body}"
241 }
[e8a22a7]242}
[5b8413b4]243
244//===========================================================================================================
245// Helper classes/variables/routines
246//===========================================================================================================
247//Description of a compiler (Must be serializable since pipelines are persistent)
248class CC_Desc implements Serializable {
[93fe3154]249 public String name
250 public String CXX
[6ebc13f]251 public String CC
[bf22bc6]252 public String lto
[93fe3154]253
[bf22bc6]254 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]255 this.name = name
256 this.CXX = CXX
[bf22bc6]257 this.CC = CC
258 this.lto = lto
[5b8413b4]259 }
260}
261
262//Description of an architecture (Must be serializable since pipelines are persistent)
263class Arch_Desc implements Serializable {
264 public String name
265 public String flags
[5307c33]266 public String node
[5b8413b4]267
[5307c33]268 Arch_Desc(String name, String flags, String node) {
[5b8413b4]269 this.name = name
270 this.flags = flags
[5307c33]271 this.node = node
[5b8413b4]272 }
273}
274
275class BuildSettings implements Serializable {
276 public final CC_Desc Compiler
277 public final Arch_Desc Architecture
[d21dd3cb]278 public final Boolean NewAST
[5b8413b4]279 public final Boolean RunAllTests
280 public final Boolean RunBenchmark
281 public final Boolean BuildDocumentation
282 public final Boolean Publish
283 public final Boolean Silent
284 public final Boolean IsSandbox
285 public final String DescLong
286 public final String DescShort
287
[a336d46]288 public String GitNewRef
289 public String GitOldRef
290
[490cb3c]291 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]292 switch( param.Compiler ) {
[099f5bd]293 case 'gcc-9':
[bf22bc6]294 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]295 break
296 case 'gcc-8':
[bf22bc6]297 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]298 break
299 case 'gcc-7':
[bf22bc6]300 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]301 break
[5b8413b4]302 case 'gcc-6':
[bf22bc6]303 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]304 break
305 case 'gcc-5':
[bf22bc6]306 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]307 break
308 case 'gcc-4.9':
[bf22bc6]309 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]310 break
311 case 'clang':
[391c065]312 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]313 break
314 default :
315 error "Unhandled compiler : ${cc}"
316 }
317
318 switch( param.Architecture ) {
319 case 'x64':
[a3e8281]320 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]321 break
322 case 'x86':
[a3e8281]323 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]324 break
325 default :
326 error "Unhandled architecture : ${arch}"
327 }
328
[f95e8f0]329 this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]330 this.NewAST = param.NewAST
[5b8413b4]331 this.RunAllTests = param.RunAllTests
[7a230fd]332 this.RunBenchmark = param.RunBenchmark
[5b8413b4]333 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]334 this.Publish = param.Publish
[5b8413b4]335 this.Silent = param.Silent
336
337 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]338 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]339
[fe3d9ab]340 final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]341 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[85799aa]342AST Version : ${ ast.toString() }
[5b8413b4]343Architecture : ${ this.Architecture.name }
344Arc Flags : ${ this.Architecture.flags }
345Run All Tests : ${ this.RunAllTests.toString() }
346Run Benchmark : ${ this.RunBenchmark.toString() }
347Build Documentation : ${ this.BuildDocumentation.toString() }
348Publish : ${ this.Publish.toString() }
349Silent : ${ this.Silent.toString() }
350"""
[a336d46]351
352 this.GitNewRef = ''
353 this.GitOldRef = ''
[5b8413b4]354 }
355}
356
[490cb3c]357class PlotGroup implements Serializable {
358 public String name
359 public String unit
360 public boolean log
361
362 PlotGroup(String name, String unit, boolean log) {
363 this.name = name
364 this.unit = unit
365 this.log = log
366 }
367}
368
[5b8413b4]369def prepare_build() {
370 // prepare the properties
371 properties ([ \
[62f96ae]372 buildDiscarder(logRotator( \
373 artifactDaysToKeepStr: '', \
374 artifactNumToKeepStr: '', \
375 daysToKeepStr: '730', \
376 numToKeepStr: '1000' \
377 )), \
[5b8413b4]378 [$class: 'ParametersDefinitionProperty', \
379 parameterDefinitions: [ \
380 [$class: 'ChoiceParameterDefinition', \
381 description: 'Which compiler to use', \
382 name: 'Compiler', \
[c09ae73]383 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]384 defaultValue: 'gcc-8', \
[5b8413b4]385 ], \
386 [$class: 'ChoiceParameterDefinition', \
387 description: 'The target architecture', \
388 name: 'Architecture', \
389 choices: 'x64\nx86', \
390 defaultValue: 'x64', \
391 ], \
[d21dd3cb]392 [$class: 'BooleanParameterDefinition', \
393 description: 'If true, build compiler using new AST', \
394 name: 'NewAST', \
[6a531ab]395 defaultValue: true, \
[d21dd3cb]396 ], \
[5b8413b4]397 [$class: 'BooleanParameterDefinition', \
398 description: 'If false, only the quick test suite is ran', \
399 name: 'RunAllTests', \
400 defaultValue: false, \
[fe3d9ab]401 ], \
[5b8413b4]402 [$class: 'BooleanParameterDefinition', \
403 description: 'If true, jenkins also runs benchmarks', \
404 name: 'RunBenchmark', \
405 defaultValue: false, \
406 ], \
407 [$class: 'BooleanParameterDefinition', \
408 description: 'If true, jenkins also builds documentation', \
409 name: 'BuildDocumentation', \
410 defaultValue: true, \
411 ], \
412 [$class: 'BooleanParameterDefinition', \
413 description: 'If true, jenkins also publishes results', \
414 name: 'Publish', \
415 defaultValue: false, \
416 ], \
417 [$class: 'BooleanParameterDefinition', \
418 description: 'If true, jenkins will not send emails', \
419 name: 'Silent', \
420 defaultValue: false, \
421 ], \
422 ],
423 ]])
424
[bd8dca2]425 // It's unfortunate but it looks like we need to checkout the entire repo just to get
426 // - the pretty git printer
427 // - Jenkins.tools
[4c55047]428 checkout scm
[2407853]429
[1483a16]430 Tools = load "Jenkins/tools.groovy"
[bd8dca2]431
[490cb3c]432 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]433
434 currentBuild.description = settings.DescShort
435 echo settings.DescLong
436
437 return settings
438}
439
440def make_doc() {
441 def err = null
442 try {
443 sh 'make clean > /dev/null'
444 sh 'make > /dev/null 2>&1'
445 }
446 catch (Exception caughtError) {
447 err = caughtError //rethrow error later
[65f4a51]448 sh 'cat build/*.log'
[5b8413b4]449 }
450 finally {
451 if (err) throw err // Must re-throw exception to propagate error
452 }
[a2a0065]453}
454
[3221a2b]455def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]456
[1b3eef8]457 if(new_data) {
458 echo "Publishing new data"
459 }
460
[cdcd53dc]461 def series = new_data ? [[
[df57a84]462 file: "${file}.csv",
[3c40dc2a]463 exclusionValues: '',
464 displayTableFlag: false,
465 inclusionFlag: 'OFF',
466 url: ''
[cdcd53dc]467 ]] : [];
[8d63649]468
469 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
470 dir("${BuildDir}/benchmark/") {
471 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
472 csvSeries: series,
[490cb3c]473 group: "${group.name}",
[3c40dc2a]474 title: "${title}",
475 style: 'lineSimple',
476 exclZero: false,
477 keepRecords: false,
[3221a2b]478 logarithmic: !relative && group.log,
[3c40dc2a]479 numBuilds: '120',
480 useDescr: true,
[490cb3c]481 yaxis: group.unit,
[3c40dc2a]482 yaxisMaximum: '',
483 yaxisMinimum: ''
[df57a84]484 }
485}
Note: See TracBrowser for help on using the repository browser.