source: Jenkinsfile@ f33eab7

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since f33eab7 was d21dd3cb, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Jenkins master build script now supports new-ast option
  • Property mode set to 100644
File size: 17.0 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5// For skipping stages
6import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
7
8//===========================================================================================================
9// Main loop of the compilation
10//===========================================================================================================
11
12node('master') {
13 // Globals
14 BuildDir = pwd tmp: true
15 SrcDir = pwd tmp: false
16 Settings = null
17 StageName = ''
18
19 // Local variables
20 def err = null
21 def log_needed = false
22
23 currentBuild.result = "SUCCESS"
24
25 try {
26 //Wrap build to add timestamp to command line
27 wrap([$class: 'TimestamperBuildWrapper']) {
28
29 Settings = prepare_build()
30
31 node(Settings.Architecture.node) {
32 BuildDir = pwd tmp: true
33 SrcDir = pwd tmp: false
34
35 clean()
36
37 checkout()
38
39 build()
40
41 test()
42
43 benchmark()
44
45 build_doc()
46
47 publish()
48 }
49
50 // Update the build directories when exiting the node
51 BuildDir = pwd tmp: true
52 SrcDir = pwd tmp: false
53 }
54 }
55
56 //If an exception is caught we need to change the status and remember to
57 //attach the build log to the email
58 catch (Exception caughtError) {
59 //rethrow error later
60 err = caughtError
61
62 echo err.toString()
63
64 //An error has occured, the build log is relevent
65 log_needed = true
66
67 //Store the result of the build log
68 currentBuild.result = "${StageName} FAILURE".trim()
69 }
70
71 finally {
72 //Send email with final results if this is not a full build
73 email(log_needed)
74
75 echo 'Build Completed'
76
77 /* Must re-throw exception to propagate error */
78 if (err) {
79 throw err
80 }
81 }
82}
83//===========================================================================================================
84// Main compilation routines
85//===========================================================================================================
86def clean() {
87 build_stage('Cleanup', true) {
88 // clean the build by wipping the build directory
89 dir(BuildDir) {
90 deleteDir()
91 }
92 }
93}
94
95//Compilation script is done here but environnement set-up and error handling is done in main loop
96def checkout() {
97 build_stage('Checkout', true) {
98 //checkout the source code and clean the repo
99 final scmVars = checkout scm
100 Settings.GitNewRef = scmVars.GIT_COMMIT
101 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
102
103 echo GitLogMessage()
104 }
105}
106
107def build() {
108 debug = true
109 release = Settings.RunAllTests || Settings.RunBenchmark
110 build_stage('Build : configure', true) {
111 // Configure must be run inside the tree
112 dir (SrcDir) {
113 // Generate the necessary build files
114 sh './autogen.sh'
115 }
116
117 // Build outside of the src tree to ease cleaning
118 dir (BuildDir) {
119 //Configure the conpilation (Output is not relevant)
120 //Use the current directory as the installation target so nothing escapes the sandbox
121 //Also specify the compiler by hand
122 targets=""
123 if( Settings.RunAllTests || Settings.RunBenchmark ) {
124 targets="--with-target-hosts='host:debug,host:nodebug'"
125 } else {
126 targets="--with-target-hosts='host:debug'"
127 }
128
129 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
130
131 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
132
133 // Configure libcfa
134 sh 'make -j 8 --no-print-directory configure-libcfa'
135 }
136 }
137
138 build_stage('Build : cfa-cpp', true) {
139 // Build outside of the src tree to ease cleaning
140 dir (BuildDir) {
141 // Build driver
142 sh 'make -j 8 --no-print-directory -C driver'
143
144 // Build translator
145 sh 'make -j 8 --no-print-directory -C src'
146 }
147 }
148
149 build_stage('Build : libcfa(debug)', debug) {
150 // Build outside of the src tree to ease cleaning
151 dir (BuildDir) {
152 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
153 }
154 }
155
156 build_stage('Build : libcfa(nodebug)', release) {
157 // Build outside of the src tree to ease cleaning
158 dir (BuildDir) {
159 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
160 }
161 }
162
163 build_stage('Build : install', true) {
164 // Build outside of the src tree to ease cleaning
165 dir (BuildDir) {
166 sh "make -j 8 --no-print-directory install"
167 }
168 }
169}
170
171def test() {
172 try {
173 build_stage('Test: short', !Settings.RunAllTests) {
174 dir (BuildDir) {
175 //Run the tests from the tests directory
176 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
177 }
178 }
179
180 build_stage('Test: full', Settings.RunAllTests) {
181 dir (BuildDir) {
182 //Run the tests from the tests directory
183 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
184 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
185 }
186 }
187 }
188 catch (Exception err) {
189 echo "Archiving core dumps"
190 dir (BuildDir) {
191 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
192 }
193 throw err
194 }
195}
196
197def benchmark() {
198 build_stage('Benchmark', Settings.RunBenchmark) {
199 dir (BuildDir) {
200 //Append bench results
201 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
202 }
203 }
204}
205
206def build_doc() {
207 build_stage('Documentation', Settings.BuildDocumentation) {
208 dir ('doc/user') {
209 make_doc()
210 }
211
212 dir ('doc/refrat') {
213 make_doc()
214 }
215 }
216}
217
218def publish() {
219 build_stage('Publish', true) {
220
221 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
222
223 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
224 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
225
226 //Then publish the results
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
232 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
233 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
234 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
235 }
236}
237
238//===========================================================================================================
239//Routine responsible of sending the email notification once the build is completed
240//===========================================================================================================
241@NonCPS
242def SplitLines(String text) {
243 def list = []
244
245 text.eachLine {
246 list += it
247 }
248
249 return list
250}
251
252def GitLogMessage() {
253 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
254
255 def oldRef = Settings.GitOldRef
256 def newRef = Settings.GitNewRef
257
258 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
259 def revList = SplitLines( revText )
260
261 def gitUpdate = ""
262 revList.each { rev ->
263 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
264 gitUpdate = gitUpdate + " via ${rev} (${type})"
265 }
266
267 def rev = oldRef
268 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
269 gitUpdate = gitUpdate + " from ${rev} (${type})"
270
271 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
272
273 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
274 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
275 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
276 gitDiff = gitDiff.replace('[m', '</span>')
277
278 return """
279<pre>
280The branch ${env.BRANCH_NAME} has been updated.
281${gitUpdate}
282</pre>
283
284<p>Check console output at ${env.BUILD_URL} to view the results.</p>
285
286<p>- Status --------------------------------------------------------------</p>
287
288<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
289
290<p>- Log -----------------------------------------------------------------</p>
291
292<pre>
293${gitLog}
294</pre>
295
296<p>-----------------------------------------------------------------------</p>
297<pre>
298Summary of changes:
299${gitDiff}
300</pre>
301"""
302}
303
304//Standard build email notification
305def email(boolean log) {
306 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
307 //Configurations for email format
308 echo 'Notifying users of result'
309
310 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
311 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
312 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
313generated because of a git hooks/post-receive script following
314a ref change which was pushed to the C\u2200 repository.</p>
315""" + GitLogMessage()
316
317 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
318
319 if( Settings && !Settings.Silent ) {
320 //send email notification
321 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
322 } else {
323 echo "Would send email to: ${email_to}"
324 echo "With title: ${email_subject}"
325 echo "Content: \n${email_body}"
326 }
327}
328
329//===========================================================================================================
330// Helper classes/variables/routines
331//===========================================================================================================
332//Description of a compiler (Must be serializable since pipelines are persistent)
333class CC_Desc implements Serializable {
334 public String name
335 public String CXX
336 public String CC
337 public String lto
338
339 CC_Desc(String name, String CXX, String CC, String lto) {
340 this.name = name
341 this.CXX = CXX
342 this.CC = CC
343 this.lto = lto
344 }
345}
346
347//Description of an architecture (Must be serializable since pipelines are persistent)
348class Arch_Desc implements Serializable {
349 public String name
350 public String flags
351 public String node
352
353 Arch_Desc(String name, String flags, String node) {
354 this.name = name
355 this.flags = flags
356 this.node = node
357 }
358}
359
360class BuildSettings implements Serializable {
361 public final CC_Desc Compiler
362 public final Arch_Desc Architecture
363 public final Boolean NewAST
364 public final Boolean RunAllTests
365 public final Boolean RunBenchmark
366 public final Boolean BuildDocumentation
367 public final Boolean Publish
368 public final Boolean Silent
369 public final Boolean IsSandbox
370 public final String DescLong
371 public final String DescShort
372
373 public String GitNewRef
374 public String GitOldRef
375
376 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
377 switch( param.Compiler ) {
378 case 'gcc-9':
379 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
380 break
381 case 'gcc-8':
382 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
383 break
384 case 'gcc-7':
385 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
386 break
387 case 'gcc-6':
388 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
389 break
390 case 'gcc-5':
391 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
392 break
393 case 'gcc-4.9':
394 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
395 break
396 case 'clang':
397 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
398 break
399 default :
400 error "Unhandled compiler : ${cc}"
401 }
402
403 switch( param.Architecture ) {
404 case 'x64':
405 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
406 break
407 case 'x86':
408 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
409 break
410 default :
411 error "Unhandled architecture : ${arch}"
412 }
413
414 this.IsSandbox = (branch == "jenkins-sandbox")
415 this.NewAST = param.NewAST
416 this.RunAllTests = param.RunAllTests
417 this.RunBenchmark = param.RunBenchmark
418 this.BuildDocumentation = param.BuildDocumentation
419 this.Publish = param.Publish
420 this.Silent = param.Silent
421
422 def full = param.RunAllTests ? " (Full)" : ""
423 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
424
425 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
426Architecture : ${ this.Architecture.name }
427Arc Flags : ${ this.Architecture.flags }
428Run All Tests : ${ this.RunAllTests.toString() }
429Run Benchmark : ${ this.RunBenchmark.toString() }
430Build Documentation : ${ this.BuildDocumentation.toString() }
431Publish : ${ this.Publish.toString() }
432Silent : ${ this.Silent.toString() }
433"""
434
435 this.GitNewRef = ''
436 this.GitOldRef = ''
437 }
438}
439
440class PlotGroup implements Serializable {
441 public String name
442 public String unit
443 public boolean log
444
445 PlotGroup(String name, String unit, boolean log) {
446 this.name = name
447 this.unit = unit
448 this.log = log
449 }
450}
451
452def prepare_build() {
453 // prepare the properties
454 properties ([ \
455 buildDiscarder(logRotator( \
456 artifactDaysToKeepStr: '', \
457 artifactNumToKeepStr: '', \
458 daysToKeepStr: '730', \
459 numToKeepStr: '1000' \
460 )), \
461 [$class: 'ParametersDefinitionProperty', \
462 parameterDefinitions: [ \
463 [$class: 'ChoiceParameterDefinition', \
464 description: 'Which compiler to use', \
465 name: 'Compiler', \
466 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
467 defaultValue: 'gcc-8', \
468 ], \
469 [$class: 'ChoiceParameterDefinition', \
470 description: 'The target architecture', \
471 name: 'Architecture', \
472 choices: 'x64\nx86', \
473 defaultValue: 'x64', \
474 ], \
475 [$class: 'BooleanParameterDefinition', \
476 description: 'If true, build compiler using new AST', \
477 name: 'NewAST', \
478 defaultValue: false, \
479 ], \
480 [$class: 'BooleanParameterDefinition', \
481 description: 'If false, only the quick test suite is ran', \
482 name: 'RunAllTests', \
483 defaultValue: false, \
484 ],
485 [$class: 'BooleanParameterDefinition', \
486 description: 'If true, jenkins also runs benchmarks', \
487 name: 'RunBenchmark', \
488 defaultValue: false, \
489 ], \
490 [$class: 'BooleanParameterDefinition', \
491 description: 'If true, jenkins also builds documentation', \
492 name: 'BuildDocumentation', \
493 defaultValue: true, \
494 ], \
495 [$class: 'BooleanParameterDefinition', \
496 description: 'If true, jenkins also publishes results', \
497 name: 'Publish', \
498 defaultValue: false, \
499 ], \
500 [$class: 'BooleanParameterDefinition', \
501 description: 'If true, jenkins will not send emails', \
502 name: 'Silent', \
503 defaultValue: false, \
504 ], \
505 ],
506 ]])
507
508 // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer
509 checkout scm
510
511 final settings = new BuildSettings(params, env.BRANCH_NAME)
512
513 currentBuild.description = settings.DescShort
514 echo settings.DescLong
515
516 return settings
517}
518
519def build_stage(String name, boolean run, Closure block ) {
520 StageName = name
521 echo " -------- ${StageName} -------- "
522 if(run) {
523 stage(name, block)
524 } else {
525 stage(name) { Utils.markStageSkippedForConditional(STAGE_NAME) }
526 }
527}
528
529def make_doc() {
530 def err = null
531 try {
532 sh 'make clean > /dev/null'
533 sh 'make > /dev/null 2>&1'
534 }
535 catch (Exception caughtError) {
536 err = caughtError //rethrow error later
537 sh 'cat build/*.log'
538 }
539 finally {
540 if (err) throw err // Must re-throw exception to propagate error
541 }
542}
543
544def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
545
546 if(new_data) {
547 echo "Publishing new data"
548 }
549
550 def series = new_data ? [[
551 file: "${file}.csv",
552 exclusionValues: '',
553 displayTableFlag: false,
554 inclusionFlag: 'OFF',
555 url: ''
556 ]] : [];
557
558 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
559 dir("${BuildDir}/benchmark/") {
560 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
561 csvSeries: series,
562 group: "${group.name}",
563 title: "${title}",
564 style: 'lineSimple',
565 exclZero: false,
566 keepRecords: false,
567 logarithmic: !relative && group.log,
568 numBuilds: '120',
569 useDescr: true,
570 yaxis: group.unit,
571 yaxisMaximum: '',
572 yaxisMinimum: ''
573 }
574}
Note: See TracBrowser for help on using the repository browser.