source: Jenkinsfile@ 8ca82de

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 8ca82de was 8ca82de, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

More attempts at loading tools
  • Property mode set to 100644
File size: 16.8 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5//===========================================================================================================
6// Main loop of the compilation
7//===========================================================================================================
8
9node('master') {
10 // Globals
11 BuildDir = pwd tmp: true
12 SrcDir = pwd tmp: false
13 Settings = null
14
15 // Local variables
16 def err = null
17 def log_needed = false
18
19 currentBuild.result = "SUCCESS"
20
21 try {
22 //Wrap build to add timestamp to command line
23 wrap([$class: 'TimestamperBuildWrapper']) {
24
25 Settings = prepare_build()
26
27 node(Settings.Architecture.node) {
28 BuildDir = pwd tmp: true
29 SrcDir = pwd tmp: false
30
31 clean()
32
33 checkout()
34
35 build()
36
37 test()
38
39 benchmark()
40
41 build_doc()
42
43 publish()
44 }
45
46 // Update the build directories when exiting the node
47 BuildDir = pwd tmp: true
48 SrcDir = pwd tmp: false
49 }
50 }
51
52 //If an exception is caught we need to change the status and remember to
53 //attach the build log to the email
54 catch (Exception caughtError) {
55 //rethrow error later
56 err = caughtError
57
58 echo err.toString()
59
60 //An error has occured, the build log is relevent
61 log_needed = true
62
63 //Store the result of the build log
64 currentBuild.result = "${tools.StageName} FAILURE".trim()
65 }
66
67 finally {
68 //Send email with final results if this is not a full build
69 email(log_needed)
70
71 echo 'Build Completed'
72
73 /* Must re-throw exception to propagate error */
74 if (err) {
75 throw err
76 }
77 }
78}
79//===========================================================================================================
80// Main compilation routines
81//===========================================================================================================
82def clean() {
83 build_stage('Cleanup', true) {
84 // clean the build by wipping the build directory
85 dir(BuildDir) {
86 deleteDir()
87 }
88 }
89}
90
91//Compilation script is done here but environnement set-up and error handling is done in main loop
92def checkout() {
93 build_stage('Checkout', true) {
94 //checkout the source code and clean the repo
95 final scmVars = checkout scm
96 Settings.GitNewRef = scmVars.GIT_COMMIT
97 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
98
99 echo GitLogMessage()
100 }
101}
102
103def build() {
104 debug = true
105 release = Settings.RunAllTests || Settings.RunBenchmark
106 build_stage('Build : configure', true) {
107 // Configure must be run inside the tree
108 dir (SrcDir) {
109 // Generate the necessary build files
110 sh './autogen.sh'
111 }
112
113 // Build outside of the src tree to ease cleaning
114 dir (BuildDir) {
115 //Configure the conpilation (Output is not relevant)
116 //Use the current directory as the installation target so nothing escapes the sandbox
117 //Also specify the compiler by hand
118 targets=""
119 if( Settings.RunAllTests || Settings.RunBenchmark ) {
120 targets="--with-target-hosts='host:debug,host:nodebug'"
121 } else {
122 targets="--with-target-hosts='host:debug'"
123 }
124
125 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
126
127 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
128
129 // Configure libcfa
130 sh 'make -j 8 --no-print-directory configure-libcfa'
131 }
132 }
133
134 build_stage('Build : cfa-cpp', true) {
135 // Build outside of the src tree to ease cleaning
136 dir (BuildDir) {
137 // Build driver
138 sh 'make -j 8 --no-print-directory -C driver'
139
140 // Build translator
141 sh 'make -j 8 --no-print-directory -C src'
142 }
143 }
144
145 build_stage('Build : libcfa(debug)', debug) {
146 // Build outside of the src tree to ease cleaning
147 dir (BuildDir) {
148 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
149 }
150 }
151
152 build_stage('Build : libcfa(nodebug)', release) {
153 // Build outside of the src tree to ease cleaning
154 dir (BuildDir) {
155 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
156 }
157 }
158
159 build_stage('Build : install', true) {
160 // Build outside of the src tree to ease cleaning
161 dir (BuildDir) {
162 sh "make -j 8 --no-print-directory install"
163 }
164 }
165}
166
167def test() {
168 try {
169 build_stage('Test: short', !Settings.RunAllTests) {
170 dir (BuildDir) {
171 //Run the tests from the tests directory
172 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
173 }
174 }
175
176 build_stage('Test: full', Settings.RunAllTests) {
177 dir (BuildDir) {
178 //Run the tests from the tests directory
179 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
180 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
181 }
182 }
183 }
184 catch (Exception err) {
185 echo "Archiving core dumps"
186 dir (BuildDir) {
187 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
188 }
189 throw err
190 }
191}
192
193def benchmark() {
194 build_stage('Benchmark', Settings.RunBenchmark) {
195 dir (BuildDir) {
196 //Append bench results
197 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
198 }
199 }
200}
201
202def build_doc() {
203 build_stage('Documentation', Settings.BuildDocumentation) {
204 dir ('doc/user') {
205 make_doc()
206 }
207
208 dir ('doc/refrat') {
209 make_doc()
210 }
211 }
212}
213
214def publish() {
215 build_stage('Publish', true) {
216
217 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
218
219 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
220 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
221
222 //Then publish the results
223 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
224 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
225 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
226 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
231 }
232}
233
234//===========================================================================================================
235//Routine responsible of sending the email notification once the build is completed
236//===========================================================================================================
237@NonCPS
238def SplitLines(String text) {
239 def list = []
240
241 text.eachLine {
242 list += it
243 }
244
245 return list
246}
247
248def GitLogMessage() {
249 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
250
251 def oldRef = Settings.GitOldRef
252 def newRef = Settings.GitNewRef
253
254 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
255 def revList = SplitLines( revText )
256
257 def gitUpdate = ""
258 revList.each { rev ->
259 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
260 gitUpdate = gitUpdate + " via ${rev} (${type})"
261 }
262
263 def rev = oldRef
264 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
265 gitUpdate = gitUpdate + " from ${rev} (${type})"
266
267 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
268
269 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
270 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
271 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
272 gitDiff = gitDiff.replace('[m', '</span>')
273
274 return """
275<pre>
276The branch ${env.BRANCH_NAME} has been updated.
277${gitUpdate}
278</pre>
279
280<p>Check console output at ${env.BUILD_URL} to view the results.</p>
281
282<p>- Status --------------------------------------------------------------</p>
283
284<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
285
286<p>- Log -----------------------------------------------------------------</p>
287
288<pre>
289${gitLog}
290</pre>
291
292<p>-----------------------------------------------------------------------</p>
293<pre>
294Summary of changes:
295${gitDiff}
296</pre>
297"""
298}
299
300//Standard build email notification
301def email(boolean log) {
302 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
303 //Configurations for email format
304 echo 'Notifying users of result'
305
306 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
307 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
308 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
309generated because of a git hooks/post-receive script following
310a ref change which was pushed to the C\u2200 repository.</p>
311""" + GitLogMessage()
312
313 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
314
315 if( Settings && !Settings.Silent ) {
316 //send email notification
317 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
318 } else {
319 echo "Would send email to: ${email_to}"
320 echo "With title: ${email_subject}"
321 echo "Content: \n${email_body}"
322 }
323}
324
325//===========================================================================================================
326// Helper classes/variables/routines
327//===========================================================================================================
328//Description of a compiler (Must be serializable since pipelines are persistent)
329class CC_Desc implements Serializable {
330 public String name
331 public String CXX
332 public String CC
333 public String lto
334
335 CC_Desc(String name, String CXX, String CC, String lto) {
336 this.name = name
337 this.CXX = CXX
338 this.CC = CC
339 this.lto = lto
340 }
341}
342
343//Description of an architecture (Must be serializable since pipelines are persistent)
344class Arch_Desc implements Serializable {
345 public String name
346 public String flags
347 public String node
348
349 Arch_Desc(String name, String flags, String node) {
350 this.name = name
351 this.flags = flags
352 this.node = node
353 }
354}
355
356class BuildSettings implements Serializable {
357 public final CC_Desc Compiler
358 public final Arch_Desc Architecture
359 public final Boolean NewAST
360 public final Boolean RunAllTests
361 public final Boolean RunBenchmark
362 public final Boolean BuildDocumentation
363 public final Boolean Publish
364 public final Boolean Silent
365 public final Boolean IsSandbox
366 public final String DescLong
367 public final String DescShort
368
369 public String GitNewRef
370 public String GitOldRef
371
372 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
373 switch( param.Compiler ) {
374 case 'gcc-9':
375 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
376 break
377 case 'gcc-8':
378 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
379 break
380 case 'gcc-7':
381 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
382 break
383 case 'gcc-6':
384 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
385 break
386 case 'gcc-5':
387 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
388 break
389 case 'gcc-4.9':
390 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
391 break
392 case 'clang':
393 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
394 break
395 default :
396 error "Unhandled compiler : ${cc}"
397 }
398
399 switch( param.Architecture ) {
400 case 'x64':
401 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
402 break
403 case 'x86':
404 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
405 break
406 default :
407 error "Unhandled architecture : ${arch}"
408 }
409
410 this.IsSandbox = (branch == "jenkins-sandbox")
411 this.NewAST = param.NewAST
412 this.RunAllTests = param.RunAllTests
413 this.RunBenchmark = param.RunBenchmark
414 this.BuildDocumentation = param.BuildDocumentation
415 this.Publish = param.Publish
416 this.Silent = param.Silent
417
418 def full = param.RunAllTests ? " (Full)" : ""
419 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
420
421 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
422Architecture : ${ this.Architecture.name }
423Arc Flags : ${ this.Architecture.flags }
424Run All Tests : ${ this.RunAllTests.toString() }
425Run Benchmark : ${ this.RunBenchmark.toString() }
426Build Documentation : ${ this.BuildDocumentation.toString() }
427Publish : ${ this.Publish.toString() }
428Silent : ${ this.Silent.toString() }
429"""
430
431 this.GitNewRef = ''
432 this.GitOldRef = ''
433 }
434}
435
436class PlotGroup implements Serializable {
437 public String name
438 public String unit
439 public boolean log
440
441 PlotGroup(String name, String unit, boolean log) {
442 this.name = name
443 this.unit = unit
444 this.log = log
445 }
446}
447
448def prepare_build() {
449 // prepare the properties
450 properties ([ \
451 buildDiscarder(logRotator( \
452 artifactDaysToKeepStr: '', \
453 artifactNumToKeepStr: '', \
454 daysToKeepStr: '730', \
455 numToKeepStr: '1000' \
456 )), \
457 [$class: 'ParametersDefinitionProperty', \
458 parameterDefinitions: [ \
459 [$class: 'ChoiceParameterDefinition', \
460 description: 'Which compiler to use', \
461 name: 'Compiler', \
462 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
463 defaultValue: 'gcc-8', \
464 ], \
465 [$class: 'ChoiceParameterDefinition', \
466 description: 'The target architecture', \
467 name: 'Architecture', \
468 choices: 'x64\nx86', \
469 defaultValue: 'x64', \
470 ], \
471 [$class: 'BooleanParameterDefinition', \
472 description: 'If true, build compiler using new AST', \
473 name: 'NewAST', \
474 defaultValue: false, \
475 ], \
476 [$class: 'BooleanParameterDefinition', \
477 description: 'If false, only the quick test suite is ran', \
478 name: 'RunAllTests', \
479 defaultValue: false, \
480 ],
481 [$class: 'BooleanParameterDefinition', \
482 description: 'If true, jenkins also runs benchmarks', \
483 name: 'RunBenchmark', \
484 defaultValue: false, \
485 ], \
486 [$class: 'BooleanParameterDefinition', \
487 description: 'If true, jenkins also builds documentation', \
488 name: 'BuildDocumentation', \
489 defaultValue: true, \
490 ], \
491 [$class: 'BooleanParameterDefinition', \
492 description: 'If true, jenkins also publishes results', \
493 name: 'Publish', \
494 defaultValue: false, \
495 ], \
496 [$class: 'BooleanParameterDefinition', \
497 description: 'If true, jenkins will not send emails', \
498 name: 'Silent', \
499 defaultValue: false, \
500 ], \
501 ],
502 ]])
503
504 // It's unfortunate but it looks like we need to checkout the entire repo just to get
505 // - the pretty git printer
506 // - Jenkins.tools
507 checkout scm
508
509 tools = load "Jenkins/tools.groovy"
510 build_stage = tools.build_stage
511
512 final settings = new BuildSettings(params, env.BRANCH_NAME)
513
514 currentBuild.description = settings.DescShort
515 echo settings.DescLong
516
517 return settings
518}
519
520def make_doc() {
521 def err = null
522 try {
523 sh 'make clean > /dev/null'
524 sh 'make > /dev/null 2>&1'
525 }
526 catch (Exception caughtError) {
527 err = caughtError //rethrow error later
528 sh 'cat build/*.log'
529 }
530 finally {
531 if (err) throw err // Must re-throw exception to propagate error
532 }
533}
534
535def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
536
537 if(new_data) {
538 echo "Publishing new data"
539 }
540
541 def series = new_data ? [[
542 file: "${file}.csv",
543 exclusionValues: '',
544 displayTableFlag: false,
545 inclusionFlag: 'OFF',
546 url: ''
547 ]] : [];
548
549 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
550 dir("${BuildDir}/benchmark/") {
551 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
552 csvSeries: series,
553 group: "${group.name}",
554 title: "${title}",
555 style: 'lineSimple',
556 exclZero: false,
557 keepRecords: false,
558 logarithmic: !relative && group.log,
559 numBuilds: '120',
560 useDescr: true,
561 yaxis: group.unit,
562 yaxisMaximum: '',
563 yaxisMinimum: ''
564 }
565}
Note: See TracBrowser for help on using the repository browser.