source: Jenkinsfile@ 68d40b7

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 68d40b7 was 7f66cec, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Re-enabled jenkins checking
  • Property mode set to 100644
File size: 17.0 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5// For skipping stages
6import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
7
8//===========================================================================================================
9// Main loop of the compilation
10//===========================================================================================================
11
12node('master') {
13 // Globals
14 BuildDir = pwd tmp: true
15 SrcDir = pwd tmp: false
16 Settings = null
17 StageName = ''
18
19 // Local variables
20 def err = null
21 def log_needed = false
22
23 currentBuild.result = "SUCCESS"
24
25 try {
26 //Wrap build to add timestamp to command line
27 wrap([$class: 'TimestamperBuildWrapper']) {
28
29 Settings = prepare_build()
30
31 node(Settings.Architecture.node) {
32 BuildDir = pwd tmp: true
33 SrcDir = pwd tmp: false
34
35 clean()
36
37 checkout()
38
39 build()
40
41 test()
42
43 benchmark()
44
45 build_doc()
46
47 publish()
48 }
49
50 // Update the build directories when exiting the node
51 BuildDir = pwd tmp: true
52 SrcDir = pwd tmp: false
53 }
54 }
55
56 //If an exception is caught we need to change the status and remember to
57 //attach the build log to the email
58 catch (Exception caughtError) {
59 //rethrow error later
60 err = caughtError
61
62 echo err.toString()
63
64 //An error has occured, the build log is relevent
65 log_needed = true
66
67 //Store the result of the build log
68 currentBuild.result = "${StageName} FAILURE".trim()
69 }
70
71 finally {
72 //Send email with final results if this is not a full build
73 email(log_needed)
74
75 echo 'Build Completed'
76
77 /* Must re-throw exception to propagate error */
78 if (err) {
79 throw err
80 }
81 }
82}
83//===========================================================================================================
84// Main compilation routines
85//===========================================================================================================
86def clean() {
87 build_stage('Cleanup', true) {
88 // clean the build by wipping the build directory
89 dir(BuildDir) {
90 deleteDir()
91 }
92 }
93}
94
95//Compilation script is done here but environnement set-up and error handling is done in main loop
96def checkout() {
97 build_stage('Checkout', true) {
98 //checkout the source code and clean the repo
99 final scmVars = checkout scm
100 Settings.GitNewRef = scmVars.GIT_COMMIT
101 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
102
103 echo GitLogMessage()
104
105 // This is a complete hack but it solves problems with automake thinking it needs to regenerate makefiles
106 // We fudged automake/missing to handle that but automake stills bakes prints inside the makefiles
107 // and these cause more problems.
108 sh 'find . -name Makefile.in -exec touch {} +'
109 }
110}
111
112def build() {
113 debug = true
114 release = Settings.RunAllTests || Settings.RunBenchmark
115 build_stage('Build : configure', true) {
116 // Configure must be run inside the tree
117 dir (SrcDir) {
118 // Generate the necessary build files
119 sh './autogen.sh'
120 }
121
122 // Build outside of the src tree to ease cleaning
123 dir (BuildDir) {
124 //Configure the conpilation (Output is not relevant)
125 //Use the current directory as the installation target so nothing escapes the sandbox
126 //Also specify the compiler by hand
127 targets=""
128 if( Settings.RunAllTests || Settings.RunBenchmark ) {
129 targets="--with-target-hosts='host:debug,host:nodebug'"
130 } else {
131 targets="--with-target-hosts='host:debug'"
132 }
133
134 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} --quiet --prefix=${BuildDir}"
135
136 // Configure libcfa
137 sh 'make -j 8 --no-print-directory configure-libcfa'
138 }
139 }
140
141 build_stage('Build : cfa-cpp', true) {
142 // Build outside of the src tree to ease cleaning
143 dir (BuildDir) {
144 // Build driver
145 sh 'make -j 8 --no-print-directory -C driver'
146
147 // Build translator
148 sh 'make -j 8 --no-print-directory -C src'
149 }
150 }
151
152 build_stage('Build : libcfa(debug)', debug) {
153 // Build outside of the src tree to ease cleaning
154 dir (BuildDir) {
155 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
156 }
157 }
158
159 build_stage('Build : libcfa(nodebug)', release) {
160 // Build outside of the src tree to ease cleaning
161 dir (BuildDir) {
162 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
163 }
164 }
165
166 build_stage('Build : install', true) {
167 // Build outside of the src tree to ease cleaning
168 dir (BuildDir) {
169 sh "make -j 8 --no-print-directory install"
170 }
171 }
172}
173
174def test() {
175 try {
176 build_stage('Test: short', !Settings.RunAllTests) {
177 dir (BuildDir) {
178 //Run the tests from the tests directory
179 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
180 }
181 }
182
183 build_stage('Test: full', Settings.RunAllTests) {
184 dir (BuildDir) {
185 //Run the tests from the tests directory
186 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
187 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
188 }
189 }
190 }
191 catch (Exception err) {
192 echo "Archiving core dumps"
193 dir (BuildDir) {
194 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
195 }
196 throw err
197 }
198}
199
200def benchmark() {
201 build_stage('Benchmark', Settings.RunBenchmark) {
202 dir (BuildDir) {
203 //Append bench results
204 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
205 }
206 }
207}
208
209def build_doc() {
210 build_stage('Documentation', Settings.BuildDocumentation) {
211 dir ('doc/user') {
212 make_doc()
213 }
214
215 dir ('doc/refrat') {
216 make_doc()
217 }
218 }
219}
220
221def publish() {
222 build_stage('Publish', true) {
223
224 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
225
226 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
227 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
228
229 //Then publish the results
230 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
231 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
232 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
233 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
234 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
235 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
236 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
237 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
238 }
239}
240
241//===========================================================================================================
242//Routine responsible of sending the email notification once the build is completed
243//===========================================================================================================
244@NonCPS
245def SplitLines(String text) {
246 def list = []
247
248 text.eachLine {
249 list += it
250 }
251
252 return list
253}
254
255def GitLogMessage() {
256 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
257
258 def oldRef = Settings.GitOldRef
259 def newRef = Settings.GitNewRef
260
261 def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim()
262 def revList = SplitLines( revText )
263
264 def gitUpdate = ""
265 revList.each { rev ->
266 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
267 gitUpdate = gitUpdate + " via ${rev} (${type})"
268 }
269
270 def rev = oldRef
271 def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim()
272 gitUpdate = gitUpdate + " from ${rev} (${type})"
273
274 def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim()
275
276 def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim()
277 gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">')
278 gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">')
279 gitDiff = gitDiff.replace('[m', '</span>')
280
281 return """
282<pre>
283The branch ${env.BRANCH_NAME} has been updated.
284${gitUpdate}
285</pre>
286
287<p>Check console output at ${env.BUILD_URL} to view the results.</p>
288
289<p>- Status --------------------------------------------------------------</p>
290
291<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
292
293<p>- Log -----------------------------------------------------------------</p>
294
295<pre>
296${gitLog}
297</pre>
298
299<p>-----------------------------------------------------------------------</p>
300<pre>
301Summary of changes:
302${gitDiff}
303</pre>
304"""
305}
306
307//Standard build email notification
308def email(boolean log) {
309 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
310 //Configurations for email format
311 echo 'Notifying users of result'
312
313 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
314 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
315 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
316generated because of a git hooks/post-receive script following
317a ref change which was pushed to the C\u2200 repository.</p>
318""" + GitLogMessage()
319
320 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
321
322 if( Settings && !Settings.Silent ) {
323 //send email notification
324 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
325 } else {
326 echo "Would send email to: ${email_to}"
327 echo "With title: ${email_subject}"
328 echo "Content: \n${email_body}"
329 }
330}
331
332//===========================================================================================================
333// Helper classes/variables/routines
334//===========================================================================================================
335//Description of a compiler (Must be serializable since pipelines are persistent)
336class CC_Desc implements Serializable {
337 public String name
338 public String CXX
339 public String CC
340 public String lto
341
342 CC_Desc(String name, String CXX, String CC, String lto) {
343 this.name = name
344 this.CXX = CXX
345 this.CC = CC
346 this.lto = lto
347 }
348}
349
350//Description of an architecture (Must be serializable since pipelines are persistent)
351class Arch_Desc implements Serializable {
352 public String name
353 public String flags
354 public String node
355
356 Arch_Desc(String name, String flags, String node) {
357 this.name = name
358 this.flags = flags
359 this.node = node
360 }
361}
362
363class BuildSettings implements Serializable {
364 public final CC_Desc Compiler
365 public final Arch_Desc Architecture
366 public final Boolean RunAllTests
367 public final Boolean RunBenchmark
368 public final Boolean BuildDocumentation
369 public final Boolean Publish
370 public final Boolean Silent
371 public final Boolean IsSandbox
372 public final String DescLong
373 public final String DescShort
374
375 public String GitNewRef
376 public String GitOldRef
377
378 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
379 switch( param.Compiler ) {
380 case 'gcc-9':
381 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
382 break
383 case 'gcc-8':
384 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
385 break
386 case 'gcc-7':
387 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
388 break
389 case 'gcc-6':
390 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
391 break
392 case 'gcc-5':
393 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
394 break
395 case 'gcc-4.9':
396 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
397 break
398 case 'clang':
399 this.Compiler = new CC_Desc('clang', 'clang++-6.0', 'gcc-6', '-flto=thin -flto-jobs=0')
400 break
401 default :
402 error "Unhandled compiler : ${cc}"
403 }
404
405 switch( param.Architecture ) {
406 case 'x64':
407 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
408 break
409 case 'x86':
410 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
411 break
412 default :
413 error "Unhandled architecture : ${arch}"
414 }
415
416 this.IsSandbox = (branch == "jenkins-sandbox")
417 this.RunAllTests = param.RunAllTests
418 this.RunBenchmark = param.RunBenchmark
419 this.BuildDocumentation = param.BuildDocumentation
420 this.Publish = param.Publish
421 this.Silent = param.Silent
422
423 def full = param.RunAllTests ? " (Full)" : ""
424 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
425
426 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
427Architecture : ${ this.Architecture.name }
428Arc Flags : ${ this.Architecture.flags }
429Run All Tests : ${ this.RunAllTests.toString() }
430Run Benchmark : ${ this.RunBenchmark.toString() }
431Build Documentation : ${ this.BuildDocumentation.toString() }
432Publish : ${ this.Publish.toString() }
433Silent : ${ this.Silent.toString() }
434"""
435
436 this.GitNewRef = ''
437 this.GitOldRef = ''
438 }
439}
440
441class PlotGroup implements Serializable {
442 public String name
443 public String unit
444 public boolean log
445
446 PlotGroup(String name, String unit, boolean log) {
447 this.name = name
448 this.unit = unit
449 this.log = log
450 }
451}
452
453def prepare_build() {
454 // prepare the properties
455 properties ([ \
456 buildDiscarder(logRotator( \
457 artifactDaysToKeepStr: '', \
458 artifactNumToKeepStr: '', \
459 daysToKeepStr: '730', \
460 numToKeepStr: '1000' \
461 )), \
462 [$class: 'ParametersDefinitionProperty', \
463 parameterDefinitions: [ \
464 [$class: 'ChoiceParameterDefinition', \
465 description: 'Which compiler to use', \
466 name: 'Compiler', \
467 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
468 defaultValue: 'gcc-8', \
469 ], \
470 [$class: 'ChoiceParameterDefinition', \
471 description: 'The target architecture', \
472 name: 'Architecture', \
473 choices: 'x64\nx86', \
474 defaultValue: 'x64', \
475 ], \
476 [$class: 'BooleanParameterDefinition', \
477 description: 'If false, only the quick test suite is ran', \
478 name: 'RunAllTests', \
479 defaultValue: false, \
480 ], \
481 [$class: 'BooleanParameterDefinition', \
482 description: 'If true, jenkins also runs benchmarks', \
483 name: 'RunBenchmark', \
484 defaultValue: false, \
485 ], \
486 [$class: 'BooleanParameterDefinition', \
487 description: 'If true, jenkins also builds documentation', \
488 name: 'BuildDocumentation', \
489 defaultValue: true, \
490 ], \
491 [$class: 'BooleanParameterDefinition', \
492 description: 'If true, jenkins also publishes results', \
493 name: 'Publish', \
494 defaultValue: false, \
495 ], \
496 [$class: 'BooleanParameterDefinition', \
497 description: 'If true, jenkins will not send emails', \
498 name: 'Silent', \
499 defaultValue: false, \
500 ], \
501 ],
502 ]])
503
504 // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer
505 checkout scm
506
507 final settings = new BuildSettings(params, env.BRANCH_NAME)
508
509 currentBuild.description = settings.DescShort
510 echo settings.DescLong
511
512 return settings
513}
514
515def build_stage(String name, boolean run, Closure block ) {
516 StageName = name
517 echo " -------- ${StageName} -------- "
518 if(run) {
519 stage(name, block)
520 } else {
521 stage(name) { Utils.markStageSkippedForConditional(STAGE_NAME) }
522 }
523}
524
525def make_doc() {
526 def err = null
527 try {
528 sh 'make clean > /dev/null'
529 sh 'make > /dev/null 2>&1'
530 }
531 catch (Exception caughtError) {
532 err = caughtError //rethrow error later
533 sh 'cat build/*.log'
534 }
535 finally {
536 if (err) throw err // Must re-throw exception to propagate error
537 }
538}
539
540def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
541
542 if(new_data) {
543 echo "Publishing new data"
544 }
545
546 def series = new_data ? [[
547 file: "${file}.csv",
548 exclusionValues: '',
549 displayTableFlag: false,
550 inclusionFlag: 'OFF',
551 url: ''
552 ]] : [];
553
554 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
555 dir("${BuildDir}/benchmark/") {
556 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
557 csvSeries: series,
558 group: "${group.name}",
559 title: "${title}",
560 style: 'lineSimple',
561 exclZero: false,
562 keepRecords: false,
563 logarithmic: !relative && group.log,
564 numBuilds: '120',
565 useDescr: true,
566 yaxis: group.unit,
567 yaxisMaximum: '',
568 yaxisMinimum: ''
569 }
570}
Note: See TracBrowser for help on using the repository browser.