source: Jenkinsfile@ d3b95f1

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since d3b95f1 was d3b95f1, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Trying new structure that uses fewer nodes.
  • Property mode set to 100644
File size: 15.1 KB
RevLine 
[a63ad80]1#!groovy
2
[7a230fd]3import groovy.transform.Field
4
[29f4fe62]5//===========================================================================================================
[9beae23]6// Main loop of the compilation
[29f4fe62]7//===========================================================================================================
[56a9ce6]8
[d3b95f1]9// Globals
10BuildDir = pwd tmp: true
11SrcDir = pwd tmp: false
12Settings = null
13Tools = null
[4f9e706]14
[d3b95f1]15// Local variables
16def err = null
17def log_needed = false
[5b8413b4]18
[d3b95f1]19currentBuild.result = "SUCCESS"
[4f9e706]20
[d3b95f1]21try {
22 //Wrap build to add timestamp to command line
23 wrap([$class: 'TimestamperBuildWrapper']) {
[0ef06b6]24
[d3b95f1]25 node {
[85142648]26 Settings = prepare_build()
[d3b95f1]27 }
[952ee7a]28
[d3b95f1]29 node(Settings.Architecture.node) {
30 BuildDir = pwd tmp: true
31 SrcDir = pwd tmp: false
32 currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
[7aebc62]33
[d3b95f1]34 Tools.Clean()
[fde808df]35
[d3b95f1]36 Tools.Checkout()
[95fdb0a]37
[d3b95f1]38 build()
[95fdb0a]39
[d3b95f1]40 test()
[95fdb0a]41
[d3b95f1]42 benchmark()
[7359098]43
[d3b95f1]44 build_doc()
[9beae23]45
[d3b95f1]46 publish()
[9beae23]47 }
[d3b95f1]48
49 // Update the build directories when exiting the node
50 BuildDir = pwd tmp: true
51 SrcDir = pwd tmp: false
[738cf8f]52 }
[d3b95f1]53}
[738cf8f]54
[d3b95f1]55//If an exception is caught we need to change the status and remember to
56//attach the build log to the email
57catch (Exception caughtError) {
58 // Store the result of the build log
59 currentBuild.result = "FAILURE"
[9dd31e7]60
[d3b95f1]61 // An error has occured, the build log is relevent
62 log_needed = true
[9beae23]63
[d3b95f1]64 // rethrow error later
65 err = caughtError
[852ae0ea]66
[d3b95f1]67 // print the error so it shows in the log
68 echo err.toString()
69}
[738cf8f]70
[d3b95f1]71finally {
72 //Send email with final results if this is not a full build
73 email(log_needed)
[9beae23]74
[d3b95f1]75 echo 'Build Completed'
[734891d]76
[d3b95f1]77 /* Must re-throw exception to propagate error */
78 if (err) {
79 throw err
[738cf8f]80 }
81}
[29f4fe62]82//===========================================================================================================
[9beae23]83// Main compilation routines
[29f4fe62]84//===========================================================================================================
[95fdb0a]85def build() {
[e507c11]86 debug = true
87 release = Settings.RunAllTests || Settings.RunBenchmark
[91aa5ab]88 Tools.BuildStage('Build : configure', true) {
[dcf1979]89 // Configure must be run inside the tree
90 dir (SrcDir) {
91 // Generate the necessary build files
92 sh './autogen.sh'
93 }
94
[50f2cfc]95 // Build outside of the src tree to ease cleaning
[5b8413b4]96 dir (BuildDir) {
[8e58264]97 //Configure the compilation (Output is not relevant)
[50f2cfc]98 //Use the current directory as the installation target so nothing escapes the sandbox
99 //Also specify the compiler by hand
[3fc5f010]100 targets=""
[d4510ea]101 if( Settings.RunAllTests || Settings.RunBenchmark ) {
[6bde81d]102 targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]103 } else {
[6bde81d]104 targets="--with-target-hosts='host:debug'"
[3fc5f010]105 }
106
[d21dd3cb]107 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
108
109 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]110
111 // Configure libcfa
[e70e54e]112 sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]113 }
114 }
115
[91aa5ab]116 Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]117 // Build outside of the src tree to ease cleaning
118 dir (BuildDir) {
119 // Build driver
120 sh 'make -j 8 --no-print-directory -C driver'
121
122 // Build translator
123 sh 'make -j 8 --no-print-directory -C src'
124 }
125 }
[1752d0e]126
[91aa5ab]127 Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]128 // Build outside of the src tree to ease cleaning
129 dir (BuildDir) {
130 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
131 }
132 }
133
[91aa5ab]134 Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]135 // Build outside of the src tree to ease cleaning
136 dir (BuildDir) {
137 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]138 }
[620dd2b]139 }
[aa96fba]140
[91aa5ab]141 Tools.BuildStage('Build : install', true) {
[aa96fba]142 // Build outside of the src tree to ease cleaning
143 dir (BuildDir) {
144 sh "make -j 8 --no-print-directory install"
145 }
146 }
[95fdb0a]147}
[24eecab]148
[95fdb0a]149def test() {
[4c1b9ea8]150 try {
[1baf6ed]151 // Print potential limits before testing
152 // in case jenkins messes with them
[a33dcd5]153 sh 'free -h'
[1baf6ed]154 sh 'ulimit -a'
155
[91aa5ab]156 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]157 dir (BuildDir) {
[3e93c00]158 //Run the tests from the tests directory
[4c51aca]159 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]160 }
[4c1b9ea8]161 }
162
[91aa5ab]163 Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]164 dir (BuildDir) {
165 //Run the tests from the tests directory
[4c51aca]166 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
167 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]168 }
[9beae23]169 }
[620dd2b]170 }
[4c1b9ea8]171 catch (Exception err) {
172 echo "Archiving core dumps"
[c95fdc9]173 dir (BuildDir) {
[cc9ec56]174 def exists = fileExists 'tests/crashes'
175 if( exists ) {
[cece53c]176 sh """${SrcDir}/tools/jenkins/archive-gen.sh"""
177 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*,setup.sh", fingerprint: true
178 }
[c95fdc9]179 }
[4c1b9ea8]180 throw err
181 }
[95fdb0a]182}
183
184def benchmark() {
[91aa5ab]185 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]186 dir (BuildDir) {
[0dc3ac3]187 //Append bench results
[c6f1f3e]188 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]189 }
[620dd2b]190 }
[9beae23]191}
[efd60d67]192
[95fdb0a]193def build_doc() {
[91aa5ab]194 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]195 dir ('doc/user') {
196 make_doc()
197 }
198
199 dir ('doc/refrat') {
200 make_doc()
201 }
[620dd2b]202 }
[9beae23]203}
204
[95fdb0a]205def publish() {
[91aa5ab]206 Tools.BuildStage('Publish', true) {
[95fdb0a]207
[1b3eef8]208 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]209
[3221a2b]210 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
211 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]212
[3898392]213 //Then publish the results
[13d2dac]214 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
215 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
216 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
217 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
218 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
219 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
220 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
221 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]222 }
[95fdb0a]223}
224
[29f4fe62]225//===========================================================================================================
226//Routine responsible of sending the email notification once the build is completed
227//===========================================================================================================
[a336d46]228//Standard build email notification
[13c98a4]229def email(boolean log) {
[a336d46]230 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
231 //Configurations for email format
232 echo 'Notifying users of result'
233
234 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
235 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]236 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]237generated because of a git hooks/post-receive script following
[986e260]238a ref change which was pushed to the C\u2200 repository.</p>
[6a531ab]239""" + Tools.GitLogMessage()
[e8a22a7]240
[13c98a4]241 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]242
[13c98a4]243 if( Settings && !Settings.Silent ) {
[094a42c]244 //send email notification
245 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
246 } else {
247 echo "Would send email to: ${email_to}"
248 echo "With title: ${email_subject}"
249 echo "Content: \n${email_body}"
250 }
[e8a22a7]251}
[5b8413b4]252
253//===========================================================================================================
254// Helper classes/variables/routines
255//===========================================================================================================
256//Description of a compiler (Must be serializable since pipelines are persistent)
257class CC_Desc implements Serializable {
[93fe3154]258 public String name
259 public String CXX
[6ebc13f]260 public String CC
[bf22bc6]261 public String lto
[93fe3154]262
[bf22bc6]263 CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]264 this.name = name
265 this.CXX = CXX
[bf22bc6]266 this.CC = CC
267 this.lto = lto
[5b8413b4]268 }
269}
270
271//Description of an architecture (Must be serializable since pipelines are persistent)
272class Arch_Desc implements Serializable {
273 public String name
274 public String flags
[5307c33]275 public String node
[5b8413b4]276
[5307c33]277 Arch_Desc(String name, String flags, String node) {
[5b8413b4]278 this.name = name
279 this.flags = flags
[5307c33]280 this.node = node
[5b8413b4]281 }
282}
283
284class BuildSettings implements Serializable {
285 public final CC_Desc Compiler
286 public final Arch_Desc Architecture
[d21dd3cb]287 public final Boolean NewAST
[5b8413b4]288 public final Boolean RunAllTests
289 public final Boolean RunBenchmark
290 public final Boolean BuildDocumentation
291 public final Boolean Publish
292 public final Boolean Silent
293 public final Boolean IsSandbox
294 public final String DescLong
295 public final String DescShort
296
[a336d46]297 public String GitNewRef
298 public String GitOldRef
299
[490cb3c]300 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]301 switch( param.Compiler ) {
[099f5bd]302 case 'gcc-9':
[bf22bc6]303 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]304 break
305 case 'gcc-8':
[bf22bc6]306 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]307 break
308 case 'gcc-7':
[bf22bc6]309 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]310 break
[5b8413b4]311 case 'gcc-6':
[bf22bc6]312 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]313 break
314 case 'gcc-5':
[bf22bc6]315 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]316 break
317 case 'gcc-4.9':
[bf22bc6]318 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]319 break
320 case 'clang':
[391c065]321 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]322 break
323 default :
324 error "Unhandled compiler : ${cc}"
325 }
326
327 switch( param.Architecture ) {
328 case 'x64':
[a3e8281]329 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]330 break
331 case 'x86':
[a3e8281]332 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]333 break
334 default :
335 error "Unhandled architecture : ${arch}"
336 }
337
[f95e8f0]338 this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]339 this.NewAST = param.NewAST
[5b8413b4]340 this.RunAllTests = param.RunAllTests
[7a230fd]341 this.RunBenchmark = param.RunBenchmark
[5b8413b4]342 this.BuildDocumentation = param.BuildDocumentation
[7a230fd]343 this.Publish = param.Publish
[5b8413b4]344 this.Silent = param.Silent
345
346 def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]347 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]348
[fe3d9ab]349 final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]350 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[85799aa]351AST Version : ${ ast.toString() }
[5b8413b4]352Architecture : ${ this.Architecture.name }
353Arc Flags : ${ this.Architecture.flags }
354Run All Tests : ${ this.RunAllTests.toString() }
355Run Benchmark : ${ this.RunBenchmark.toString() }
356Build Documentation : ${ this.BuildDocumentation.toString() }
357Publish : ${ this.Publish.toString() }
358Silent : ${ this.Silent.toString() }
359"""
[a336d46]360
361 this.GitNewRef = ''
362 this.GitOldRef = ''
[5b8413b4]363 }
364}
365
[490cb3c]366class PlotGroup implements Serializable {
367 public String name
368 public String unit
369 public boolean log
370
371 PlotGroup(String name, String unit, boolean log) {
372 this.name = name
373 this.unit = unit
374 this.log = log
375 }
376}
377
[5b8413b4]378def prepare_build() {
379 // prepare the properties
380 properties ([ \
[62f96ae]381 buildDiscarder(logRotator( \
382 artifactDaysToKeepStr: '', \
383 artifactNumToKeepStr: '', \
384 daysToKeepStr: '730', \
385 numToKeepStr: '1000' \
386 )), \
[5b8413b4]387 [$class: 'ParametersDefinitionProperty', \
388 parameterDefinitions: [ \
389 [$class: 'ChoiceParameterDefinition', \
390 description: 'Which compiler to use', \
391 name: 'Compiler', \
[c09ae73]392 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]393 defaultValue: 'gcc-8', \
[5b8413b4]394 ], \
395 [$class: 'ChoiceParameterDefinition', \
396 description: 'The target architecture', \
397 name: 'Architecture', \
398 choices: 'x64\nx86', \
399 defaultValue: 'x64', \
400 ], \
[d21dd3cb]401 [$class: 'BooleanParameterDefinition', \
402 description: 'If true, build compiler using new AST', \
403 name: 'NewAST', \
[6a531ab]404 defaultValue: true, \
[d21dd3cb]405 ], \
[5b8413b4]406 [$class: 'BooleanParameterDefinition', \
407 description: 'If false, only the quick test suite is ran', \
408 name: 'RunAllTests', \
409 defaultValue: false, \
[fe3d9ab]410 ], \
[5b8413b4]411 [$class: 'BooleanParameterDefinition', \
412 description: 'If true, jenkins also runs benchmarks', \
413 name: 'RunBenchmark', \
414 defaultValue: false, \
415 ], \
416 [$class: 'BooleanParameterDefinition', \
417 description: 'If true, jenkins also builds documentation', \
418 name: 'BuildDocumentation', \
419 defaultValue: true, \
420 ], \
421 [$class: 'BooleanParameterDefinition', \
422 description: 'If true, jenkins also publishes results', \
423 name: 'Publish', \
424 defaultValue: false, \
425 ], \
426 [$class: 'BooleanParameterDefinition', \
427 description: 'If true, jenkins will not send emails', \
428 name: 'Silent', \
429 defaultValue: false, \
430 ], \
431 ],
432 ]])
433
[bd8dca2]434 // It's unfortunate but it looks like we need to checkout the entire repo just to get
435 // - the pretty git printer
436 // - Jenkins.tools
[4c55047]437 checkout scm
[2407853]438
[1483a16]439 Tools = load "Jenkins/tools.groovy"
[bd8dca2]440
[490cb3c]441 final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]442
443 currentBuild.description = settings.DescShort
444 echo settings.DescLong
445
446 return settings
447}
448
449def make_doc() {
450 def err = null
451 try {
452 sh 'make clean > /dev/null'
453 sh 'make > /dev/null 2>&1'
454 }
455 catch (Exception caughtError) {
456 err = caughtError //rethrow error later
[65f4a51]457 sh 'cat build/*.log'
[5b8413b4]458 }
459 finally {
460 if (err) throw err // Must re-throw exception to propagate error
461 }
[a2a0065]462}
463
[3221a2b]464def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]465
[1b3eef8]466 if(new_data) {
467 echo "Publishing new data"
468 }
469
[cdcd53dc]470 def series = new_data ? [[
[df57a84]471 file: "${file}.csv",
[3c40dc2a]472 exclusionValues: '',
473 displayTableFlag: false,
474 inclusionFlag: 'OFF',
475 url: ''
[cdcd53dc]476 ]] : [];
[8d63649]477
478 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
479 dir("${BuildDir}/benchmark/") {
480 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
481 csvSeries: series,
[490cb3c]482 group: "${group.name}",
[3c40dc2a]483 title: "${title}",
484 style: 'lineSimple',
485 exclZero: false,
486 keepRecords: false,
[3221a2b]487 logarithmic: !relative && group.log,
[3c40dc2a]488 numBuilds: '120',
489 useDescr: true,
[490cb3c]490 yaxis: group.unit,
[3c40dc2a]491 yaxisMaximum: '',
492 yaxisMinimum: ''
[df57a84]493 }
494}
Note: See TracBrowser for help on using the repository browser.