source: Jenkinsfile@ 953d433

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 953d433 was 953d433, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

More attempts at loading tools
  • Property mode set to 100644
File size: 16.8 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5//===========================================================================================================
6// Main loop of the compilation
7//===========================================================================================================
8
9node('master') {
10 // Globals
11 BuildDir = pwd tmp: true
12 SrcDir = pwd tmp: false
13 Settings = null
14 tools = null
15 build_stage = null
16
17 // Local variables
18 def err = null
19 def log_needed = false
20
21 currentBuild.result = "SUCCESS"
22
23 try {
24 //Wrap build to add timestamp to command line
25 wrap([$class: 'TimestamperBuildWrapper']) {
26
27 Settings = prepare_build()
28
29 node(Settings.Architecture.node) {
30 BuildDir = pwd tmp: true
31 SrcDir = pwd tmp: false
32
33 clean()
34
35 checkout()
36
37 build()
38
39 test()
40
41 benchmark()
42
43 build_doc()
44
45 publish()
46 }
47
48 // Update the build directories when exiting the node
49 BuildDir = pwd tmp: true
50 SrcDir = pwd tmp: false
51 }
52 }
53
54 //If an exception is caught we need to change the status and remember to
55 //attach the build log to the email
56 catch (Exception caughtError) {
57 //rethrow error later
58 err = caughtError
59
60 echo err.toString()
61
62 //An error has occured, the build log is relevent
63 log_needed = true
64
65 //Store the result of the build log
66 currentBuild.result = "${tools.StageName} FAILURE".trim()
67 }
68
69 finally {
70 //Send email with final results if this is not a full build
71 email(log_needed)
72
73 echo 'Build Completed'
74
75 /* Must re-throw exception to propagate error */
76 if (err) {
77 throw err
78 }
79 }
80}
81//===========================================================================================================
82// Main compilation routines
83//===========================================================================================================
84def clean() {
85 build_stage('Cleanup', true) {
86 // clean the build by wipping the build directory
87 dir(BuildDir) {
88 deleteDir()
89 }
90 }
91}
92
93//Compilation script is done here but environnement set-up and error handling is done in main loop
94def checkout() {
95 build_stage('Checkout', true) {
96 //checkout the source code and clean the repo
97 final scmVars = checkout scm
98 Settings.GitNewRef = scmVars.GIT_COMMIT
99 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
100
101 echo GitLogMessage()
102 }
103}
104
105def build() {
106 debug = true
107 release = Settings.RunAllTests || Settings.RunBenchmark
108 build_stage('Build : configure', true) {
109 // Configure must be run inside the tree
110 dir (SrcDir) {
111 // Generate the necessary build files
112 sh './autogen.sh'
113 }
114
115 // Build outside of the src tree to ease cleaning
116 dir (BuildDir) {
117 //Configure the conpilation (Output is not relevant)
118 //Use the current directory as the installation target so nothing escapes the sandbox
119 //Also specify the compiler by hand
120 targets=""
121 if( Settings.RunAllTests || Settings.RunBenchmark ) {
122 targets="--with-target-hosts='host:debug,host:nodebug'"
123 } else {
124 targets="--with-target-hosts='host:debug'"
125 }
126
127 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
128
129 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
130
131 // Configure libcfa
132 sh 'make -j 8 --no-print-directory configure-libcfa'
133 }
134 }
135
136 build_stage('Build : cfa-cpp', true) {
137 // Build outside of the src tree to ease cleaning
138 dir (BuildDir) {
139 // Build driver
140 sh 'make -j 8 --no-print-directory -C driver'
141
142 // Build translator
143 sh 'make -j 8 --no-print-directory -C src'
144 }
145 }
146
147 build_stage('Build : libcfa(debug)', debug) {
148 // Build outside of the src tree to ease cleaning
149 dir (BuildDir) {
150 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
151 }
152 }
153
154 build_stage('Build : libcfa(nodebug)', release) {
155 // Build outside of the src tree to ease cleaning
156 dir (BuildDir) {
157 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
158 }
159 }
160
161 build_stage('Build : install', true) {
162 // Build outside of the src tree to ease cleaning
163 dir (BuildDir) {
164 sh "make -j 8 --no-print-directory install"
165 }
166 }
167}
168
169def test() {
170 try {
171 build_stage('Test: short', !Settings.RunAllTests) {
172 dir (BuildDir) {
173 //Run the tests from the tests directory
174 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
175 }
176 }
177
178 build_stage('Test: full', Settings.RunAllTests) {
179 dir (BuildDir) {
180 //Run the tests from the tests directory
181 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
182 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
183 }
184 }
185 }
186 catch (Exception err) {
187 echo "Archiving core dumps"
188 dir (BuildDir) {
189 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
190 }
191 throw err
192 }
193}
194
195def benchmark() {
196 build_stage('Benchmark', Settings.RunBenchmark) {
197 dir (BuildDir) {
198 //Append bench results
199 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
200 }
201 }
202}
203
204def build_doc() {
205 build_stage('Documentation', Settings.BuildDocumentation) {
206 dir ('doc/user') {
207 make_doc()
208 }
209
210 dir ('doc/refrat') {
211 make_doc()
212 }
213 }
214}
215
216def publish() {
217 build_stage('Publish', true) {
218
219 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
220
221 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
222 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
223
224 //Then publish the results
225 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
226 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
232 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
233 }
234}
235
236//===========================================================================================================
237//Routine responsible of sending the email notification once the build is completed
238//===========================================================================================================
239@NonCPS
240def SplitLines(String text) {
241 def list = []
242
243 text.eachLine {
244 list += it
245 }
246
247 return list
248}
249
250def GitLogMessage() {
251 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
252
253 def oldRef = Settings.GitOldRef
254 def newRef = Settings.GitNewRef
255
256 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
257 def revList = SplitLines( revText )
258
259 def gitUpdate = ""
260 revList.each { rev ->
261 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
262 gitUpdate = gitUpdate + " via ${rev} (${type})"
263 }
264
265 def rev = oldRef
266 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
267 gitUpdate = gitUpdate + " from ${rev} (${type})"
268
269 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
270
271 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
272 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
273 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
274 gitDiff = gitDiff.replace('[m', '</span>')
275
276 return """
277<pre>
278The branch ${env.BRANCH_NAME} has been updated.
279${gitUpdate}
280</pre>
281
282<p>Check console output at ${env.BUILD_URL} to view the results.</p>
283
284<p>- Status --------------------------------------------------------------</p>
285
286<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
287
288<p>- Log -----------------------------------------------------------------</p>
289
290<pre>
291${gitLog}
292</pre>
293
294<p>-----------------------------------------------------------------------</p>
295<pre>
296Summary of changes:
297${gitDiff}
298</pre>
299"""
300}
301
302//Standard build email notification
303def email(boolean log) {
304 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
305 //Configurations for email format
306 echo 'Notifying users of result'
307
308 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
309 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
310 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
311generated because of a git hooks/post-receive script following
312a ref change which was pushed to the C\u2200 repository.</p>
313""" + GitLogMessage()
314
315 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
316
317 if( Settings && !Settings.Silent ) {
318 //send email notification
319 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
320 } else {
321 echo "Would send email to: ${email_to}"
322 echo "With title: ${email_subject}"
323 echo "Content: \n${email_body}"
324 }
325}
326
327//===========================================================================================================
328// Helper classes/variables/routines
329//===========================================================================================================
330//Description of a compiler (Must be serializable since pipelines are persistent)
331class CC_Desc implements Serializable {
332 public String name
333 public String CXX
334 public String CC
335 public String lto
336
337 CC_Desc(String name, String CXX, String CC, String lto) {
338 this.name = name
339 this.CXX = CXX
340 this.CC = CC
341 this.lto = lto
342 }
343}
344
345//Description of an architecture (Must be serializable since pipelines are persistent)
346class Arch_Desc implements Serializable {
347 public String name
348 public String flags
349 public String node
350
351 Arch_Desc(String name, String flags, String node) {
352 this.name = name
353 this.flags = flags
354 this.node = node
355 }
356}
357
358class BuildSettings implements Serializable {
359 public final CC_Desc Compiler
360 public final Arch_Desc Architecture
361 public final Boolean NewAST
362 public final Boolean RunAllTests
363 public final Boolean RunBenchmark
364 public final Boolean BuildDocumentation
365 public final Boolean Publish
366 public final Boolean Silent
367 public final Boolean IsSandbox
368 public final String DescLong
369 public final String DescShort
370
371 public String GitNewRef
372 public String GitOldRef
373
374 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
375 switch( param.Compiler ) {
376 case 'gcc-9':
377 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
378 break
379 case 'gcc-8':
380 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
381 break
382 case 'gcc-7':
383 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
384 break
385 case 'gcc-6':
386 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
387 break
388 case 'gcc-5':
389 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
390 break
391 case 'gcc-4.9':
392 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
393 break
394 case 'clang':
395 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
396 break
397 default :
398 error "Unhandled compiler : ${cc}"
399 }
400
401 switch( param.Architecture ) {
402 case 'x64':
403 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
404 break
405 case 'x86':
406 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
407 break
408 default :
409 error "Unhandled architecture : ${arch}"
410 }
411
412 this.IsSandbox = (branch == "jenkins-sandbox")
413 this.NewAST = param.NewAST
414 this.RunAllTests = param.RunAllTests
415 this.RunBenchmark = param.RunBenchmark
416 this.BuildDocumentation = param.BuildDocumentation
417 this.Publish = param.Publish
418 this.Silent = param.Silent
419
420 def full = param.RunAllTests ? " (Full)" : ""
421 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
422
423 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
424Architecture : ${ this.Architecture.name }
425Arc Flags : ${ this.Architecture.flags }
426Run All Tests : ${ this.RunAllTests.toString() }
427Run Benchmark : ${ this.RunBenchmark.toString() }
428Build Documentation : ${ this.BuildDocumentation.toString() }
429Publish : ${ this.Publish.toString() }
430Silent : ${ this.Silent.toString() }
431"""
432
433 this.GitNewRef = ''
434 this.GitOldRef = ''
435 }
436}
437
438class PlotGroup implements Serializable {
439 public String name
440 public String unit
441 public boolean log
442
443 PlotGroup(String name, String unit, boolean log) {
444 this.name = name
445 this.unit = unit
446 this.log = log
447 }
448}
449
450def prepare_build() {
451 // prepare the properties
452 properties ([ \
453 buildDiscarder(logRotator( \
454 artifactDaysToKeepStr: '', \
455 artifactNumToKeepStr: '', \
456 daysToKeepStr: '730', \
457 numToKeepStr: '1000' \
458 )), \
459 [$class: 'ParametersDefinitionProperty', \
460 parameterDefinitions: [ \
461 [$class: 'ChoiceParameterDefinition', \
462 description: 'Which compiler to use', \
463 name: 'Compiler', \
464 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
465 defaultValue: 'gcc-8', \
466 ], \
467 [$class: 'ChoiceParameterDefinition', \
468 description: 'The target architecture', \
469 name: 'Architecture', \
470 choices: 'x64\nx86', \
471 defaultValue: 'x64', \
472 ], \
473 [$class: 'BooleanParameterDefinition', \
474 description: 'If true, build compiler using new AST', \
475 name: 'NewAST', \
476 defaultValue: false, \
477 ], \
478 [$class: 'BooleanParameterDefinition', \
479 description: 'If false, only the quick test suite is ran', \
480 name: 'RunAllTests', \
481 defaultValue: false, \
482 ],
483 [$class: 'BooleanParameterDefinition', \
484 description: 'If true, jenkins also runs benchmarks', \
485 name: 'RunBenchmark', \
486 defaultValue: false, \
487 ], \
488 [$class: 'BooleanParameterDefinition', \
489 description: 'If true, jenkins also builds documentation', \
490 name: 'BuildDocumentation', \
491 defaultValue: true, \
492 ], \
493 [$class: 'BooleanParameterDefinition', \
494 description: 'If true, jenkins also publishes results', \
495 name: 'Publish', \
496 defaultValue: false, \
497 ], \
498 [$class: 'BooleanParameterDefinition', \
499 description: 'If true, jenkins will not send emails', \
500 name: 'Silent', \
501 defaultValue: false, \
502 ], \
503 ],
504 ]])
505
506 // It's unfortunate but it looks like we need to checkout the entire repo just to get
507 // - the pretty git printer
508 // - Jenkins.tools
509 checkout scm
510
511 tools = load "Jenkins/tools.groovy"
512 build_stage = tools.build_stage
513
514 final settings = new BuildSettings(params, env.BRANCH_NAME)
515
516 currentBuild.description = settings.DescShort
517 echo settings.DescLong
518
519 return settings
520}
521
522def make_doc() {
523 def err = null
524 try {
525 sh 'make clean > /dev/null'
526 sh 'make > /dev/null 2>&1'
527 }
528 catch (Exception caughtError) {
529 err = caughtError //rethrow error later
530 sh 'cat build/*.log'
531 }
532 finally {
533 if (err) throw err // Must re-throw exception to propagate error
534 }
535}
536
537def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
538
539 if(new_data) {
540 echo "Publishing new data"
541 }
542
543 def series = new_data ? [[
544 file: "${file}.csv",
545 exclusionValues: '',
546 displayTableFlag: false,
547 inclusionFlag: 'OFF',
548 url: ''
549 ]] : [];
550
551 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
552 dir("${BuildDir}/benchmark/") {
553 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
554 csvSeries: series,
555 group: "${group.name}",
556 title: "${title}",
557 style: 'lineSimple',
558 exclZero: false,
559 keepRecords: false,
560 logarithmic: !relative && group.log,
561 numBuilds: '120',
562 useDescr: true,
563 yaxis: group.unit,
564 yaxisMaximum: '',
565 yaxisMinimum: ''
566 }
567}
Note: See TracBrowser for help on using the repository browser.