source: Jenkinsfile@ 391c065

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 391c065 was 391c065, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Update jenkins to use more recent version of clang
  • Property mode set to 100644
File size: 16.7 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5// For skipping stages
6import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
7
8//===========================================================================================================
9// Main loop of the compilation
10//===========================================================================================================
11
12node('master') {
13 // Globals
14 BuildDir = pwd tmp: true
15 SrcDir = pwd tmp: false
16 Settings = null
17 StageName = ''
18
19 // Local variables
20 def err = null
21 def log_needed = false
22
23 currentBuild.result = "SUCCESS"
24
25 try {
26 //Wrap build to add timestamp to command line
27 wrap([$class: 'TimestamperBuildWrapper']) {
28
29 Settings = prepare_build()
30
31 node(Settings.Architecture.node) {
32 BuildDir = pwd tmp: true
33 SrcDir = pwd tmp: false
34
35 clean()
36
37 checkout()
38
39 build()
40
41 test()
42
43 benchmark()
44
45 build_doc()
46
47 publish()
48 }
49
50 // Update the build directories when exiting the node
51 BuildDir = pwd tmp: true
52 SrcDir = pwd tmp: false
53 }
54 }
55
56 //If an exception is caught we need to change the status and remember to
57 //attach the build log to the email
58 catch (Exception caughtError) {
59 //rethrow error later
60 err = caughtError
61
62 echo err.toString()
63
64 //An error has occured, the build log is relevent
65 log_needed = true
66
67 //Store the result of the build log
68 currentBuild.result = "${StageName} FAILURE".trim()
69 }
70
71 finally {
72 //Send email with final results if this is not a full build
73 email(log_needed)
74
75 echo 'Build Completed'
76
77 /* Must re-throw exception to propagate error */
78 if (err) {
79 throw err
80 }
81 }
82}
83//===========================================================================================================
84// Main compilation routines
85//===========================================================================================================
86def clean() {
87 build_stage('Cleanup', true) {
88 // clean the build by wipping the build directory
89 dir(BuildDir) {
90 deleteDir()
91 }
92 }
93}
94
95//Compilation script is done here but environnement set-up and error handling is done in main loop
96def checkout() {
97 build_stage('Checkout', true) {
98 //checkout the source code and clean the repo
99 final scmVars = checkout scm
100 Settings.GitNewRef = scmVars.GIT_COMMIT
101 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
102
103 echo GitLogMessage()
104 }
105}
106
107def build() {
108 debug = true
109 release = Settings.RunAllTests || Settings.RunBenchmark
110 build_stage('Build : configure', true) {
111 // Configure must be run inside the tree
112 dir (SrcDir) {
113 // Generate the necessary build files
114 sh './autogen.sh'
115 }
116
117 // Build outside of the src tree to ease cleaning
118 dir (BuildDir) {
119 //Configure the conpilation (Output is not relevant)
120 //Use the current directory as the installation target so nothing escapes the sandbox
121 //Also specify the compiler by hand
122 targets=""
123 if( Settings.RunAllTests || Settings.RunBenchmark ) {
124 targets="--with-target-hosts='host:debug,host:nodebug'"
125 } else {
126 targets="--with-target-hosts='host:debug'"
127 }
128
129 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} --quiet --prefix=${BuildDir}"
130
131 // Configure libcfa
132 sh 'make -j 8 --no-print-directory configure-libcfa'
133 }
134 }
135
136 build_stage('Build : cfa-cpp', true) {
137 // Build outside of the src tree to ease cleaning
138 dir (BuildDir) {
139 // Build driver
140 sh 'make -j 8 --no-print-directory -C driver'
141
142 // Build translator
143 sh 'make -j 8 --no-print-directory -C src'
144 }
145 }
146
147 build_stage('Build : libcfa(debug)', debug) {
148 // Build outside of the src tree to ease cleaning
149 dir (BuildDir) {
150 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
151 }
152 }
153
154 build_stage('Build : libcfa(nodebug)', release) {
155 // Build outside of the src tree to ease cleaning
156 dir (BuildDir) {
157 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
158 }
159 }
160
161 build_stage('Build : install', true) {
162 // Build outside of the src tree to ease cleaning
163 dir (BuildDir) {
164 sh "make -j 8 --no-print-directory install"
165 }
166 }
167}
168
169def test() {
170 try {
171 build_stage('Test: short', !Settings.RunAllTests) {
172 dir (BuildDir) {
173 //Run the tests from the tests directory
174 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
175 }
176 }
177
178 build_stage('Test: full', Settings.RunAllTests) {
179 dir (BuildDir) {
180 //Run the tests from the tests directory
181 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
182 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
183 }
184 }
185 }
186 catch (Exception err) {
187 echo "Archiving core dumps"
188 dir (BuildDir) {
189 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
190 }
191 throw err
192 }
193}
194
195def benchmark() {
196 build_stage('Benchmark', Settings.RunBenchmark) {
197 dir (BuildDir) {
198 //Append bench results
199 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
200 }
201 }
202}
203
204def build_doc() {
205 build_stage('Documentation', Settings.BuildDocumentation) {
206 dir ('doc/user') {
207 make_doc()
208 }
209
210 dir ('doc/refrat') {
211 make_doc()
212 }
213 }
214}
215
216def publish() {
217 build_stage('Publish', true) {
218
219 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
220
221 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
222 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
223
224 //Then publish the results
225 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
226 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
227 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
228 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
229 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
232 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
233 }
234}
235
236//===========================================================================================================
237//Routine responsible of sending the email notification once the build is completed
238//===========================================================================================================
239@NonCPS
240def SplitLines(String text) {
241 def list = []
242
243 text.eachLine {
244 list += it
245 }
246
247 return list
248}
249
250def GitLogMessage() {
251 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
252
253 def oldRef = Settings.GitOldRef
254 def newRef = Settings.GitNewRef
255
256 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
257 def revList = SplitLines( revText )
258
259 def gitUpdate = ""
260 revList.each { rev ->
261 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
262 gitUpdate = gitUpdate + " via ${rev} (${type})"
263 }
264
265 def rev = oldRef
266 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
267 gitUpdate = gitUpdate + " from ${rev} (${type})"
268
269 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
270
271 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
272 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
273 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
274 gitDiff = gitDiff.replace('[m', '</span>')
275
276 return """
277<pre>
278The branch ${env.BRANCH_NAME} has been updated.
279${gitUpdate}
280</pre>
281
282<p>Check console output at ${env.BUILD_URL} to view the results.</p>
283
284<p>- Status --------------------------------------------------------------</p>
285
286<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
287
288<p>- Log -----------------------------------------------------------------</p>
289
290<pre>
291${gitLog}
292</pre>
293
294<p>-----------------------------------------------------------------------</p>
295<pre>
296Summary of changes:
297${gitDiff}
298</pre>
299"""
300}
301
302//Standard build email notification
303def email(boolean log) {
304 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
305 //Configurations for email format
306 echo 'Notifying users of result'
307
308 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
309 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
310 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
311generated because of a git hooks/post-receive script following
312a ref change which was pushed to the C\u2200 repository.</p>
313""" + GitLogMessage()
314
315 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
316
317 if( Settings && !Settings.Silent ) {
318 //send email notification
319 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
320 } else {
321 echo "Would send email to: ${email_to}"
322 echo "With title: ${email_subject}"
323 echo "Content: \n${email_body}"
324 }
325}
326
327//===========================================================================================================
328// Helper classes/variables/routines
329//===========================================================================================================
330//Description of a compiler (Must be serializable since pipelines are persistent)
331class CC_Desc implements Serializable {
332 public String name
333 public String CXX
334 public String CC
335 public String lto
336
337 CC_Desc(String name, String CXX, String CC, String lto) {
338 this.name = name
339 this.CXX = CXX
340 this.CC = CC
341 this.lto = lto
342 }
343}
344
345//Description of an architecture (Must be serializable since pipelines are persistent)
346class Arch_Desc implements Serializable {
347 public String name
348 public String flags
349 public String node
350
351 Arch_Desc(String name, String flags, String node) {
352 this.name = name
353 this.flags = flags
354 this.node = node
355 }
356}
357
358class BuildSettings implements Serializable {
359 public final CC_Desc Compiler
360 public final Arch_Desc Architecture
361 public final Boolean RunAllTests
362 public final Boolean RunBenchmark
363 public final Boolean BuildDocumentation
364 public final Boolean Publish
365 public final Boolean Silent
366 public final Boolean IsSandbox
367 public final String DescLong
368 public final String DescShort
369
370 public String GitNewRef
371 public String GitOldRef
372
373 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
374 switch( param.Compiler ) {
375 case 'gcc-9':
376 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
377 break
378 case 'gcc-8':
379 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
380 break
381 case 'gcc-7':
382 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
383 break
384 case 'gcc-6':
385 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
386 break
387 case 'gcc-5':
388 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
389 break
390 case 'gcc-4.9':
391 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
392 break
393 case 'clang':
394 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
395 break
396 default :
397 error "Unhandled compiler : ${cc}"
398 }
399
400 switch( param.Architecture ) {
401 case 'x64':
402 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
403 break
404 case 'x86':
405 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
406 break
407 default :
408 error "Unhandled architecture : ${arch}"
409 }
410
411 this.IsSandbox = (branch == "jenkins-sandbox")
412 this.RunAllTests = param.RunAllTests
413 this.RunBenchmark = param.RunBenchmark
414 this.BuildDocumentation = param.BuildDocumentation
415 this.Publish = param.Publish
416 this.Silent = param.Silent
417
418 def full = param.RunAllTests ? " (Full)" : ""
419 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
420
421 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
422Architecture : ${ this.Architecture.name }
423Arc Flags : ${ this.Architecture.flags }
424Run All Tests : ${ this.RunAllTests.toString() }
425Run Benchmark : ${ this.RunBenchmark.toString() }
426Build Documentation : ${ this.BuildDocumentation.toString() }
427Publish : ${ this.Publish.toString() }
428Silent : ${ this.Silent.toString() }
429"""
430
431 this.GitNewRef = ''
432 this.GitOldRef = ''
433 }
434}
435
436class PlotGroup implements Serializable {
437 public String name
438 public String unit
439 public boolean log
440
441 PlotGroup(String name, String unit, boolean log) {
442 this.name = name
443 this.unit = unit
444 this.log = log
445 }
446}
447
448def prepare_build() {
449 // prepare the properties
450 properties ([ \
451 buildDiscarder(logRotator( \
452 artifactDaysToKeepStr: '', \
453 artifactNumToKeepStr: '', \
454 daysToKeepStr: '730', \
455 numToKeepStr: '1000' \
456 )), \
457 [$class: 'ParametersDefinitionProperty', \
458 parameterDefinitions: [ \
459 [$class: 'ChoiceParameterDefinition', \
460 description: 'Which compiler to use', \
461 name: 'Compiler', \
462 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
463 defaultValue: 'gcc-8', \
464 ], \
465 [$class: 'ChoiceParameterDefinition', \
466 description: 'The target architecture', \
467 name: 'Architecture', \
468 choices: 'x64\nx86', \
469 defaultValue: 'x64', \
470 ], \
471 [$class: 'BooleanParameterDefinition', \
472 description: 'If false, only the quick test suite is ran', \
473 name: 'RunAllTests', \
474 defaultValue: false, \
475 ], \
476 [$class: 'BooleanParameterDefinition', \
477 description: 'If true, jenkins also runs benchmarks', \
478 name: 'RunBenchmark', \
479 defaultValue: false, \
480 ], \
481 [$class: 'BooleanParameterDefinition', \
482 description: 'If true, jenkins also builds documentation', \
483 name: 'BuildDocumentation', \
484 defaultValue: true, \
485 ], \
486 [$class: 'BooleanParameterDefinition', \
487 description: 'If true, jenkins also publishes results', \
488 name: 'Publish', \
489 defaultValue: false, \
490 ], \
491 [$class: 'BooleanParameterDefinition', \
492 description: 'If true, jenkins will not send emails', \
493 name: 'Silent', \
494 defaultValue: false, \
495 ], \
496 ],
497 ]])
498
499 // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer
500 checkout scm
501
502 final settings = new BuildSettings(params, env.BRANCH_NAME)
503
504 currentBuild.description = settings.DescShort
505 echo settings.DescLong
506
507 return settings
508}
509
510def build_stage(String name, boolean run, Closure block ) {
511 StageName = name
512 echo " -------- ${StageName} -------- "
513 if(run) {
514 stage(name, block)
515 } else {
516 stage(name) { Utils.markStageSkippedForConditional(STAGE_NAME) }
517 }
518}
519
520def make_doc() {
521 def err = null
522 try {
523 sh 'make clean > /dev/null'
524 sh 'make > /dev/null 2>&1'
525 }
526 catch (Exception caughtError) {
527 err = caughtError //rethrow error later
528 sh 'cat build/*.log'
529 }
530 finally {
531 if (err) throw err // Must re-throw exception to propagate error
532 }
533}
534
535def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
536
537 if(new_data) {
538 echo "Publishing new data"
539 }
540
541 def series = new_data ? [[
542 file: "${file}.csv",
543 exclusionValues: '',
544 displayTableFlag: false,
545 inclusionFlag: 'OFF',
546 url: ''
547 ]] : [];
548
549 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
550 dir("${BuildDir}/benchmark/") {
551 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
552 csvSeries: series,
553 group: "${group.name}",
554 title: "${title}",
555 style: 'lineSimple',
556 exclZero: false,
557 keepRecords: false,
558 logarithmic: !relative && group.log,
559 numBuilds: '120',
560 useDescr: true,
561 yaxis: group.unit,
562 yaxisMaximum: '',
563 yaxisMinimum: ''
564 }
565}
Note: See TracBrowser for help on using the repository browser.