source: Jenkinsfile@ 27b1ca1

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 27b1ca1 was 27b1ca1, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Modified Jenkinsfile to move out some of the helper for later reuse
  • Property mode set to 100644
File size: 16.8 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5import static Jenkins.tools.build_stage as build_stage
6import Jenkins.tools
7
8//===========================================================================================================
9// Main loop of the compilation
10//===========================================================================================================
11
12node('master') {
13 // Globals
14 BuildDir = pwd tmp: true
15 SrcDir = pwd tmp: false
16 Settings = null
17
18 // Local variables
19 def err = null
20 def log_needed = false
21
22 currentBuild.result = "SUCCESS"
23
24 try {
25 //Wrap build to add timestamp to command line
26 wrap([$class: 'TimestamperBuildWrapper']) {
27
28 Settings = prepare_build()
29
30 node(Settings.Architecture.node) {
31 BuildDir = pwd tmp: true
32 SrcDir = pwd tmp: false
33
34 clean()
35
36 checkout()
37
38 build()
39
40 test()
41
42 benchmark()
43
44 build_doc()
45
46 publish()
47 }
48
49 // Update the build directories when exiting the node
50 BuildDir = pwd tmp: true
51 SrcDir = pwd tmp: false
52 }
53 }
54
55 //If an exception is caught we need to change the status and remember to
56 //attach the build log to the email
57 catch (Exception caughtError) {
58 //rethrow error later
59 err = caughtError
60
61 echo err.toString()
62
63 //An error has occured, the build log is relevent
64 log_needed = true
65
66 //Store the result of the build log
67 currentBuild.result = "${tools.StageName} FAILURE".trim()
68 }
69
70 finally {
71 //Send email with final results if this is not a full build
72 email(log_needed)
73
74 echo 'Build Completed'
75
76 /* Must re-throw exception to propagate error */
77 if (err) {
78 throw err
79 }
80 }
81}
82//===========================================================================================================
83// Main compilation routines
84//===========================================================================================================
85def clean() {
86 build_stage('Cleanup', true) {
87 // clean the build by wipping the build directory
88 dir(BuildDir) {
89 deleteDir()
90 }
91 }
92}
93
94//Compilation script is done here but environnement set-up and error handling is done in main loop
95def checkout() {
96 build_stage('Checkout', true) {
97 //checkout the source code and clean the repo
98 final scmVars = checkout scm
99 Settings.GitNewRef = scmVars.GIT_COMMIT
100 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
101
102 echo GitLogMessage()
103 }
104}
105
106def build() {
107 debug = true
108 release = Settings.RunAllTests || Settings.RunBenchmark
109 build_stage('Build : configure', true) {
110 // Configure must be run inside the tree
111 dir (SrcDir) {
112 // Generate the necessary build files
113 sh './autogen.sh'
114 }
115
116 // Build outside of the src tree to ease cleaning
117 dir (BuildDir) {
118 //Configure the conpilation (Output is not relevant)
119 //Use the current directory as the installation target so nothing escapes the sandbox
120 //Also specify the compiler by hand
121 targets=""
122 if( Settings.RunAllTests || Settings.RunBenchmark ) {
123 targets="--with-target-hosts='host:debug,host:nodebug'"
124 } else {
125 targets="--with-target-hosts='host:debug'"
126 }
127
128 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
129
130 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
131
132 // Configure libcfa
133 sh 'make -j 8 --no-print-directory configure-libcfa'
134 }
135 }
136
137 build_stage('Build : cfa-cpp', true) {
138 // Build outside of the src tree to ease cleaning
139 dir (BuildDir) {
140 // Build driver
141 sh 'make -j 8 --no-print-directory -C driver'
142
143 // Build translator
144 sh 'make -j 8 --no-print-directory -C src'
145 }
146 }
147
148 build_stage('Build : libcfa(debug)', debug) {
149 // Build outside of the src tree to ease cleaning
150 dir (BuildDir) {
151 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
152 }
153 }
154
155 build_stage('Build : libcfa(nodebug)', release) {
156 // Build outside of the src tree to ease cleaning
157 dir (BuildDir) {
158 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
159 }
160 }
161
162 build_stage('Build : install', true) {
163 // Build outside of the src tree to ease cleaning
164 dir (BuildDir) {
165 sh "make -j 8 --no-print-directory install"
166 }
167 }
168}
169
170def test() {
171 try {
172 build_stage('Test: short', !Settings.RunAllTests) {
173 dir (BuildDir) {
174 //Run the tests from the tests directory
175 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
176 }
177 }
178
179 build_stage('Test: full', Settings.RunAllTests) {
180 dir (BuildDir) {
181 //Run the tests from the tests directory
182 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
183 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
184 }
185 }
186 }
187 catch (Exception err) {
188 echo "Archiving core dumps"
189 dir (BuildDir) {
190 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
191 }
192 throw err
193 }
194}
195
196def benchmark() {
197 build_stage('Benchmark', Settings.RunBenchmark) {
198 dir (BuildDir) {
199 //Append bench results
200 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
201 }
202 }
203}
204
205def build_doc() {
206 build_stage('Documentation', Settings.BuildDocumentation) {
207 dir ('doc/user') {
208 make_doc()
209 }
210
211 dir ('doc/refrat') {
212 make_doc()
213 }
214 }
215}
216
217def publish() {
218 build_stage('Publish', true) {
219
220 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
221
222 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
223 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
224
225 //Then publish the results
226 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
232 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
233 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
234 }
235}
236
237//===========================================================================================================
238//Routine responsible of sending the email notification once the build is completed
239//===========================================================================================================
240@NonCPS
241def SplitLines(String text) {
242 def list = []
243
244 text.eachLine {
245 list += it
246 }
247
248 return list
249}
250
251def GitLogMessage() {
252 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
253
254 def oldRef = Settings.GitOldRef
255 def newRef = Settings.GitNewRef
256
257 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
258 def revList = SplitLines( revText )
259
260 def gitUpdate = ""
261 revList.each { rev ->
262 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
263 gitUpdate = gitUpdate + " via ${rev} (${type})"
264 }
265
266 def rev = oldRef
267 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
268 gitUpdate = gitUpdate + " from ${rev} (${type})"
269
270 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
271
272 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
273 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
274 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
275 gitDiff = gitDiff.replace('[m', '</span>')
276
277 return """
278<pre>
279The branch ${env.BRANCH_NAME} has been updated.
280${gitUpdate}
281</pre>
282
283<p>Check console output at ${env.BUILD_URL} to view the results.</p>
284
285<p>- Status --------------------------------------------------------------</p>
286
287<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
288
289<p>- Log -----------------------------------------------------------------</p>
290
291<pre>
292${gitLog}
293</pre>
294
295<p>-----------------------------------------------------------------------</p>
296<pre>
297Summary of changes:
298${gitDiff}
299</pre>
300"""
301}
302
303//Standard build email notification
304def email(boolean log) {
305 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
306 //Configurations for email format
307 echo 'Notifying users of result'
308
309 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
310 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
311 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
312generated because of a git hooks/post-receive script following
313a ref change which was pushed to the C\u2200 repository.</p>
314""" + GitLogMessage()
315
316 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
317
318 if( Settings && !Settings.Silent ) {
319 //send email notification
320 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
321 } else {
322 echo "Would send email to: ${email_to}"
323 echo "With title: ${email_subject}"
324 echo "Content: \n${email_body}"
325 }
326}
327
328//===========================================================================================================
329// Helper classes/variables/routines
330//===========================================================================================================
331//Description of a compiler (Must be serializable since pipelines are persistent)
332class CC_Desc implements Serializable {
333 public String name
334 public String CXX
335 public String CC
336 public String lto
337
338 CC_Desc(String name, String CXX, String CC, String lto) {
339 this.name = name
340 this.CXX = CXX
341 this.CC = CC
342 this.lto = lto
343 }
344}
345
346//Description of an architecture (Must be serializable since pipelines are persistent)
347class Arch_Desc implements Serializable {
348 public String name
349 public String flags
350 public String node
351
352 Arch_Desc(String name, String flags, String node) {
353 this.name = name
354 this.flags = flags
355 this.node = node
356 }
357}
358
359class BuildSettings implements Serializable {
360 public final CC_Desc Compiler
361 public final Arch_Desc Architecture
362 public final Boolean NewAST
363 public final Boolean RunAllTests
364 public final Boolean RunBenchmark
365 public final Boolean BuildDocumentation
366 public final Boolean Publish
367 public final Boolean Silent
368 public final Boolean IsSandbox
369 public final String DescLong
370 public final String DescShort
371
372 public String GitNewRef
373 public String GitOldRef
374
375 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
376 switch( param.Compiler ) {
377 case 'gcc-9':
378 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
379 break
380 case 'gcc-8':
381 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
382 break
383 case 'gcc-7':
384 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
385 break
386 case 'gcc-6':
387 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
388 break
389 case 'gcc-5':
390 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
391 break
392 case 'gcc-4.9':
393 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
394 break
395 case 'clang':
396 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
397 break
398 default :
399 error "Unhandled compiler : ${cc}"
400 }
401
402 switch( param.Architecture ) {
403 case 'x64':
404 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
405 break
406 case 'x86':
407 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
408 break
409 default :
410 error "Unhandled architecture : ${arch}"
411 }
412
413 this.IsSandbox = (branch == "jenkins-sandbox")
414 this.NewAST = param.NewAST
415 this.RunAllTests = param.RunAllTests
416 this.RunBenchmark = param.RunBenchmark
417 this.BuildDocumentation = param.BuildDocumentation
418 this.Publish = param.Publish
419 this.Silent = param.Silent
420
421 def full = param.RunAllTests ? " (Full)" : ""
422 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
423
424 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
425Architecture : ${ this.Architecture.name }
426Arc Flags : ${ this.Architecture.flags }
427Run All Tests : ${ this.RunAllTests.toString() }
428Run Benchmark : ${ this.RunBenchmark.toString() }
429Build Documentation : ${ this.BuildDocumentation.toString() }
430Publish : ${ this.Publish.toString() }
431Silent : ${ this.Silent.toString() }
432"""
433
434 this.GitNewRef = ''
435 this.GitOldRef = ''
436 }
437}
438
439class PlotGroup implements Serializable {
440 public String name
441 public String unit
442 public boolean log
443
444 PlotGroup(String name, String unit, boolean log) {
445 this.name = name
446 this.unit = unit
447 this.log = log
448 }
449}
450
451def prepare_build() {
452 // prepare the properties
453 properties ([ \
454 buildDiscarder(logRotator( \
455 artifactDaysToKeepStr: '', \
456 artifactNumToKeepStr: '', \
457 daysToKeepStr: '730', \
458 numToKeepStr: '1000' \
459 )), \
460 [$class: 'ParametersDefinitionProperty', \
461 parameterDefinitions: [ \
462 [$class: 'ChoiceParameterDefinition', \
463 description: 'Which compiler to use', \
464 name: 'Compiler', \
465 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
466 defaultValue: 'gcc-8', \
467 ], \
468 [$class: 'ChoiceParameterDefinition', \
469 description: 'The target architecture', \
470 name: 'Architecture', \
471 choices: 'x64\nx86', \
472 defaultValue: 'x64', \
473 ], \
474 [$class: 'BooleanParameterDefinition', \
475 description: 'If true, build compiler using new AST', \
476 name: 'NewAST', \
477 defaultValue: false, \
478 ], \
479 [$class: 'BooleanParameterDefinition', \
480 description: 'If false, only the quick test suite is ran', \
481 name: 'RunAllTests', \
482 defaultValue: false, \
483 ],
484 [$class: 'BooleanParameterDefinition', \
485 description: 'If true, jenkins also runs benchmarks', \
486 name: 'RunBenchmark', \
487 defaultValue: false, \
488 ], \
489 [$class: 'BooleanParameterDefinition', \
490 description: 'If true, jenkins also builds documentation', \
491 name: 'BuildDocumentation', \
492 defaultValue: true, \
493 ], \
494 [$class: 'BooleanParameterDefinition', \
495 description: 'If true, jenkins also publishes results', \
496 name: 'Publish', \
497 defaultValue: false, \
498 ], \
499 [$class: 'BooleanParameterDefinition', \
500 description: 'If true, jenkins will not send emails', \
501 name: 'Silent', \
502 defaultValue: false, \
503 ], \
504 ],
505 ]])
506
507 // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer
508 checkout scm
509
510 final settings = new BuildSettings(params, env.BRANCH_NAME)
511
512 currentBuild.description = settings.DescShort
513 echo settings.DescLong
514
515 return settings
516}
517
518def make_doc() {
519 def err = null
520 try {
521 sh 'make clean > /dev/null'
522 sh 'make > /dev/null 2>&1'
523 }
524 catch (Exception caughtError) {
525 err = caughtError //rethrow error later
526 sh 'cat build/*.log'
527 }
528 finally {
529 if (err) throw err // Must re-throw exception to propagate error
530 }
531}
532
533def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
534
535 if(new_data) {
536 echo "Publishing new data"
537 }
538
539 def series = new_data ? [[
540 file: "${file}.csv",
541 exclusionValues: '',
542 displayTableFlag: false,
543 inclusionFlag: 'OFF',
544 url: ''
545 ]] : [];
546
547 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
548 dir("${BuildDir}/benchmark/") {
549 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
550 csvSeries: series,
551 group: "${group.name}",
552 title: "${title}",
553 style: 'lineSimple',
554 exclZero: false,
555 keepRecords: false,
556 logarithmic: !relative && group.log,
557 numBuilds: '120',
558 useDescr: true,
559 yaxis: group.unit,
560 yaxisMaximum: '',
561 yaxisMinimum: ''
562 }
563}
Note: See TracBrowser for help on using the repository browser.