source: Jenkinsfile@ 7684557

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 7684557 was 7684557, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Build description now cotains node name
  • Property mode set to 100644
File size: 15.1 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[29f4fe62]5//===========================================================================================================
[9beae23]6// Main loop of the compilation
[29f4fe62]7//===========================================================================================================
[56a9ce6]8
[85142648]9node('master') {
10 // Globals
11 BuildDir = pwd tmp: true
12 SrcDir = pwd tmp: false
13 Settings = null
[1483a16]14 Tools = null
[4f9e706]15
[85142648]16 // Local variables
17 def err = null
18 def log_needed = false
[5b8413b4]19
[85142648]20 currentBuild.result = "SUCCESS"
[4f9e706]21
[85142648]22 try {
23 //Wrap build to add timestamp to command line
24 wrap([$class: 'TimestamperBuildWrapper']) {
[0ef06b6]25
[85142648]26 Settings = prepare_build()
[952ee7a]27
[85142648]28 node(Settings.Architecture.node) {
[9dd31e7]29 BuildDir = pwd tmp: true
30 SrcDir = pwd tmp: false
[7684557]31 currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
[23a14d86]32
[9dd31e7]33 Tools.Clean()
[7aebc62]34
[9dd31e7]35 Tools.Checkout()
[fde808df]36
[9dd31e7]37 build()
[95fdb0a]38
[9dd31e7]39 test()
[95fdb0a]40
[9dd31e7]41 benchmark()
[95fdb0a]42
[9dd31e7]43 build_doc()
[7359098]44
[9dd31e7]45 publish()
[85142648]46 }
[9beae23]47
[85142648]48 // Update the build directories when exiting the node
49 BuildDir = pwd tmp: true
50 SrcDir = pwd tmp: false
[9beae23]51 }
[738cf8f]52 }
53
[85142648]54 //If an exception is caught we need to change the status and remember to
55 //attach the build log to the email
56 catch (Exception caughtError) {
[9dd31e7]57 // Store the result of the build log
58 currentBuild.result = "FAILURE"
59
60 // An error has occured, the build log is relevent
[85142648]61 log_needed = true
[9beae23]62
[9dd31e7]63 // rethrow error later
[852ae0ea]64 err = caughtError
65
[9dd31e7]66 // print the error so it shows in the log
[852ae0ea]67 echo err.toString()
[85142648]68 }
[738cf8f]69
[85142648]70 finally {
71 //Send email with final results if this is not a full build
72 email(log_needed)
[9beae23]73
[85142648]74 echo 'Build Completed'
[734891d]75
[85142648]76 /* Must re-throw exception to propagate error */
77 if (err) {
78 throw err
79 }
[738cf8f]80 }
81}
[29f4fe62]82//===========================================================================================================
[9beae23]83// Main compilation routines
[29f4fe62]84//===========================================================================================================
[95fdb0a]85def build() {
[e507c11]86 debug = true
87 release = Settings.RunAllTests || Settings.RunBenchmark
[91aa5ab]88 Tools.BuildStage('Build : configure', true) {
[dcf1979]89 // Configure must be run inside the tree
90 dir (SrcDir) {
91 // Generate the necessary build files
92 sh './autogen.sh'
93 }
94
[50f2cfc]95 // Build outside of the src tree to ease cleaning
[5b8413b4]96 dir (BuildDir) {
[8e58264]97 //Configure the compilation (Output is not relevant)
[50f2cfc]98 //Use the current directory as the installation target so nothing escapes the sandbox
99 //Also specify the compiler by hand
[3fc5f010]100 targets=""
[d4510ea]101 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]102 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]103 } else {
[6bde81d]104 targets="--with-target-hosts='host:debug'"
[3fc5f010]105 }
106
[d21dd3cb]107 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
108
109 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]110
111 // Configure libcfa
[e70e54e]112 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]113 }
114 }
115
[91aa5ab]116 Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]117 // Build outside of the src tree to ease cleaning
118 dir (BuildDir) {
119 // Build driver
120 sh 'make -j 8 --no-print-directory -C driver'
121
122 // Build translator
123 sh 'make -j 8 --no-print-directory -C src'
124 }
125 }
[1752d0e]126
[91aa5ab]127 Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]128 // Build outside of the src tree to ease cleaning
129 dir (BuildDir) {
130 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
131 }
132 }
133
[91aa5ab]134 Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]135 // Build outside of the src tree to ease cleaning
136 dir (BuildDir) {
137 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]138 }
[620dd2b]139 }
[aa96fba]140
[91aa5ab]141 Tools.BuildStage('Build : install', true) {
[aa96fba]142 // Build outside of the src tree to ease cleaning
143 dir (BuildDir) {
144 sh "make -j 8 --no-print-directory install"
145 }
146 }
[95fdb0a]147}
[24eecab]148
[95fdb0a]149def test() {
[4c1b9ea8]150 try {
[1baf6ed]151 // Print potential limits before testing
152 // in case jenkins messes with them
[a33dcd5]153 sh 'free -h'
[1baf6ed]154 sh 'ulimit -a'
155
[91aa5ab]156 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]157 dir (BuildDir) {
[3e93c00]158 //Run the tests from the tests directory
[4c51aca]159 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]160 }
[4c1b9ea8]161 }
162
[91aa5ab]163 Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]164 dir (BuildDir) {
165 //Run the tests from the tests directory
[4c51aca]166 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
167 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]168 }
[9beae23]169 }
[620dd2b]170 }
[4c1b9ea8]171 catch (Exception err) {
172 echo "Archiving core dumps"
[c95fdc9]173 dir (BuildDir) {
[32ab519]174 sh """${SrcDir}/tools/jenkins/archive-gen.sh"""
175 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*,setup.sh", fingerprint: true
[c95fdc9]176 }
[4c1b9ea8]177 throw err
178 }
[95fdb0a]179}
180
181def benchmark() {
[91aa5ab]182 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]183 dir (BuildDir) {
[0dc3ac3]184 //Append bench results
[c6f1f3e]185 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]186 }
[620dd2b]187 }
[9beae23]188}
[efd60d67]189
[95fdb0a]190def build_doc() {
[91aa5ab]191 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]192 dir ('doc/user') {
193 make_doc()
194 }
195
196 dir ('doc/refrat') {
197 make_doc()
198 }
[620dd2b]199 }
[9beae23]200}
201
[95fdb0a]202def publish() {
[91aa5ab]203 Tools.BuildStage('Publish', true) {
[95fdb0a]204
[1b3eef8]205 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]206
[3221a2b]207 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
208 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]209
[3898392]210 //Then publish the results
[13d2dac]211 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
212 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
213 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
214 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
215 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
216 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
217 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
218 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]219 }
[95fdb0a]220}
221
[29f4fe62]222//===========================================================================================================
223//Routine responsible of sending the email notification once the build is completed
224//===========================================================================================================
[a336d46]225//Standard build email notification
[13c98a4]226def email(boolean log) {
[a336d46]227 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
228 //Configurations for email format
229 echo 'Notifying users of result'
230
231 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
232 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]233 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]234generated because of a git hooks/post-receive script following
[986e260]235a ref change which was pushed to the C\u2200 repository.</p>
[6a531ab]236""" + Tools.GitLogMessage()
[e8a22a7]237
[13c98a4]238 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]239
[13c98a4]240 if( Settings && !Settings.Silent ) {
[094a42c]241 //send email notification
242 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
243 } else {
244 echo "Would send email to: ${email_to}"
245 echo "With title: ${email_subject}"
246 echo "Content: \n${email_body}"
247 }
[e8a22a7]248}
[5b8413b4]249
250//===========================================================================================================
251// Helper classes/variables/routines
252//===========================================================================================================
253//Description of a compiler (Must be serializable since pipelines are persistent)
254class CC_Desc implements Serializable {
[93fe3154]255 public String name
256 public String CXX
[6ebc13f]257 public String CC
[bf22bc6]258 public String lto
[93fe3154]259
[bf22bc6]260 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]261 this.name = name
262 this.CXX = CXX
[bf22bc6]263 this.CC = CC
264 this.lto = lto
[5b8413b4]265 }
266}
267
268//Description of an architecture (Must be serializable since pipelines are persistent)
269class Arch_Desc implements Serializable {
270 public String name
271 public String flags
[5307c33]272 public String node
[5b8413b4]273
[5307c33]274 Arch_Desc(String name, String flags, String node) {
[5b8413b4]275 this.name = name
276 this.flags = flags
[5307c33]277 this.node = node
[5b8413b4]278 }
279}
280
281class BuildSettings implements Serializable {
282 public final CC_Desc Compiler
283 public final Arch_Desc Architecture
[d21dd3cb]284 public final Boolean NewAST
[5b8413b4]285 public final Boolean RunAllTests
286 public final Boolean RunBenchmark
287 public final Boolean BuildDocumentation
288 public final Boolean Publish
289 public final Boolean Silent
290 public final Boolean IsSandbox
291 public final String DescLong
292 public final String DescShort
293
[a336d46]294 public String GitNewRef
295 public String GitOldRef
296
[490cb3c]297 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]298 switch( param.Compiler ) {
[099f5bd]299 case 'gcc-9':
[bf22bc6]300 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]301 break
302 case 'gcc-8':
[bf22bc6]303 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]304 break
305 case 'gcc-7':
[bf22bc6]306 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]307 break
[5b8413b4]308 case 'gcc-6':
[bf22bc6]309 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]310 break
311 case 'gcc-5':
[bf22bc6]312 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]313 break
314 case 'gcc-4.9':
[bf22bc6]315 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]316 break
317 case 'clang':
[391c065]318 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]319 break
320 default :
321 error "Unhandled compiler : ${cc}"
322 }
323
324 switch( param.Architecture ) {
325 case 'x64':
[a3e8281]326 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]327 break
328 case 'x86':
[a3e8281]329 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]330 break
331 default :
332 error "Unhandled architecture : ${arch}"
333 }
334
[f95e8f0]335 this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]336 this.NewAST = param.NewAST
[5b8413b4]337 this.RunAllTests = param.RunAllTests
[7a230fd]338 this.RunBenchmark = param.RunBenchmark
[5b8413b4]339 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]340 this.Publish = param.Publish
[5b8413b4]341 this.Silent = param.Silent
342
343 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]344 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]345
[fe3d9ab]346 final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]347 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[85799aa]348AST Version : ${ ast.toString() }
[5b8413b4]349Architecture : ${ this.Architecture.name }
350Arc Flags : ${ this.Architecture.flags }
351Run All Tests : ${ this.RunAllTests.toString() }
352Run Benchmark : ${ this.RunBenchmark.toString() }
353Build Documentation : ${ this.BuildDocumentation.toString() }
354Publish : ${ this.Publish.toString() }
355Silent : ${ this.Silent.toString() }
356"""
[a336d46]357
358 this.GitNewRef = ''
359 this.GitOldRef = ''
[5b8413b4]360 }
361}
362
[490cb3c]363class PlotGroup implements Serializable {
364 public String name
365 public String unit
366 public boolean log
367
368 PlotGroup(String name, String unit, boolean log) {
369 this.name = name
370 this.unit = unit
371 this.log = log
372 }
373}
374
[5b8413b4]375def prepare_build() {
376 // prepare the properties
377 properties ([ \
[62f96ae]378 buildDiscarder(logRotator( \
379 artifactDaysToKeepStr: '', \
380 artifactNumToKeepStr: '', \
381 daysToKeepStr: '730', \
382 numToKeepStr: '1000' \
383 )), \
[5b8413b4]384 [$class: 'ParametersDefinitionProperty', \
385 parameterDefinitions: [ \
386 [$class: 'ChoiceParameterDefinition', \
387 description: 'Which compiler to use', \
388 name: 'Compiler', \
[c09ae73]389 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]390 defaultValue: 'gcc-8', \
[5b8413b4]391 ], \
392 [$class: 'ChoiceParameterDefinition', \
393 description: 'The target architecture', \
394 name: 'Architecture', \
395 choices: 'x64\nx86', \
396 defaultValue: 'x64', \
397 ], \
[d21dd3cb]398 [$class: 'BooleanParameterDefinition', \
399 description: 'If true, build compiler using new AST', \
400 name: 'NewAST', \
[6a531ab]401 defaultValue: true, \
[d21dd3cb]402 ], \
[5b8413b4]403 [$class: 'BooleanParameterDefinition', \
404 description: 'If false, only the quick test suite is ran', \
405 name: 'RunAllTests', \
406 defaultValue: false, \
[fe3d9ab]407 ], \
[5b8413b4]408 [$class: 'BooleanParameterDefinition', \
409 description: 'If true, jenkins also runs benchmarks', \
410 name: 'RunBenchmark', \
411 defaultValue: false, \
412 ], \
413 [$class: 'BooleanParameterDefinition', \
414 description: 'If true, jenkins also builds documentation', \
415 name: 'BuildDocumentation', \
416 defaultValue: true, \
417 ], \
418 [$class: 'BooleanParameterDefinition', \
419 description: 'If true, jenkins also publishes results', \
420 name: 'Publish', \
421 defaultValue: false, \
422 ], \
423 [$class: 'BooleanParameterDefinition', \
424 description: 'If true, jenkins will not send emails', \
425 name: 'Silent', \
426 defaultValue: false, \
427 ], \
428 ],
429 ]])
430
[bd8dca2]431 // It's unfortunate but it looks like we need to checkout the entire repo just to get
432 // - the pretty git printer
433 // - Jenkins.tools
[4c55047]434 checkout scm
[2407853]435
[1483a16]436 Tools = load "Jenkins/tools.groovy"
[bd8dca2]437
[490cb3c]438 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]439
440 currentBuild.description = settings.DescShort
441 echo settings.DescLong
442
443 return settings
444}
445
446def make_doc() {
447 def err = null
448 try {
449 sh 'make clean > /dev/null'
450 sh 'make > /dev/null 2>&1'
451 }
452 catch (Exception caughtError) {
453 err = caughtError //rethrow error later
[65f4a51]454 sh 'cat build/*.log'
[5b8413b4]455 }
456 finally {
457 if (err) throw err // Must re-throw exception to propagate error
458 }
[a2a0065]459}
460
[3221a2b]461def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]462
[1b3eef8]463 if(new_data) {
464 echo "Publishing new data"
465 }
466
[cdcd53dc]467 def series = new_data ? [[
[df57a84]468 file: "${file}.csv",
[3c40dc2a]469 exclusionValues: '',
470 displayTableFlag: false,
471 inclusionFlag: 'OFF',
472 url: ''
[cdcd53dc]473 ]] : [];
[8d63649]474
475 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
476 dir("${BuildDir}/benchmark/") {
477 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
478 csvSeries: series,
[490cb3c]479 group: "${group.name}",
[3c40dc2a]480 title: "${title}",
481 style: 'lineSimple',
482 exclZero: false,
483 keepRecords: false,
[3221a2b]484 logarithmic: !relative && group.log,
[3c40dc2a]485 numBuilds: '120',
486 useDescr: true,
[490cb3c]487 yaxis: group.unit,
[3c40dc2a]488 yaxisMaximum: '',
489 yaxisMinimum: ''
[df57a84]490 }
491}
Note: See TracBrowser for help on using the repository browser.