source: Jenkinsfile@ 0d92e5c

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation jenkins-sandbox new-ast new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 0d92e5c was c6f1f3e, checked in by Thierry Delisle <tdelisle@…>, 6 years ago

Jenkins now also computes various speed-up graphs for performance monitoring

  • Property mode set to 100644
File size: 14.5 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5// For skipping stages
6import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
7
8//===========================================================================================================
9// Main loop of the compilation
10//===========================================================================================================
11
12node('master') {
13 // Globals
14 BuildDir = pwd tmp: true
15 SrcDir = pwd tmp: false
16 Settings = null
17 StageName = ''
18
19 // Local variables
20 def err = null
21 def log_needed = false
22
23 currentBuild.result = "SUCCESS"
24
25 try {
26 //Wrap build to add timestamp to command line
27 wrap([$class: 'TimestamperBuildWrapper']) {
28
29 Settings = prepare_build()
30
31 node(Settings.Architecture.node) {
32 BuildDir = pwd tmp: true
33 SrcDir = pwd tmp: false
34
35 clean()
36
37 checkout()
38
39 build()
40
41 test()
42
43 benchmark()
44
45 build_doc()
46
47 publish()
48 }
49
50 // Update the build directories when exiting the node
51 BuildDir = pwd tmp: true
52 SrcDir = pwd tmp: false
53 }
54 }
55
56 //If an exception is caught we need to change the status and remember to
57 //attach the build log to the email
58 catch (Exception caughtError) {
59 //rethrow error later
60 err = caughtError
61
62 echo err.toString()
63
64 //An error has occured, the build log is relevent
65 log_needed = true
66
67 //Store the result of the build log
68 currentBuild.result = "${StageName} FAILURE".trim()
69 }
70
71 finally {
72 //Send email with final results if this is not a full build
73 email(log_needed)
74
75 echo 'Build Completed'
76
77 /* Must re-throw exception to propagate error */
78 if (err) {
79 throw err
80 }
81 }
82}
83//===========================================================================================================
84// Main compilation routines
85//===========================================================================================================
86def clean() {
87 build_stage('Cleanup', true) {
88 // clean the build by wipping the build directory
89 dir(BuildDir) {
90 deleteDir()
91 }
92 }
93}
94
95//Compilation script is done here but environnement set-up and error handling is done in main loop
96def checkout() {
97 build_stage('Checkout', true) {
98 //checkout the source code and clean the repo
99 final scmVars = checkout scm
100 Settings.GitNewRef = scmVars.GIT_COMMIT
101 Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT
102
103 echo GitLogMessage()
104 }
105}
106
107def build() {
108 debug = true
109 release = Settings.RunAllTests || Settings.RunBenchmark
110 build_stage('Build : configure', true) {
111 // Build outside of the src tree to ease cleaning
112 dir (BuildDir) {
113 //Configure the conpilation (Output is not relevant)
114 //Use the current directory as the installation target so nothing escapes the sandbox
115 //Also specify the compiler by hand
116 targets=""
117 if( Settings.RunAllTests || Settings.RunBenchmark ) {
118 targets="--with-target-hosts='host:debug,host:nodebug'"
119 } else {
120 targets="--with-target-hosts='host:debug'"
121 }
122
123 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} ${targets} --quiet"
124
125 // Configure libcfa
126 sh 'make -j 8 --no-print-directory configure-libcfa'
127 }
128 }
129
130 build_stage('Build : cfa-cpp', true) {
131 // Build outside of the src tree to ease cleaning
132 dir (BuildDir) {
133 // Build driver
134 sh 'make -j 8 --no-print-directory -C driver'
135
136 // Build translator
137 sh 'make -j 8 --no-print-directory -C src'
138 }
139 }
140
141 build_stage('Build : libcfa(debug)', debug) {
142 // Build outside of the src tree to ease cleaning
143 dir (BuildDir) {
144 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
145 }
146 }
147
148 build_stage('Build : libcfa(nodebug)', release) {
149 // Build outside of the src tree to ease cleaning
150 dir (BuildDir) {
151 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
152 }
153 }
154}
155
156def test() {
157 build_stage('Test: short', !Settings.RunAllTests) {
158 dir (BuildDir) {
159 //Run the tests from the tests directory
160 sh 'make --no-print-directory -C tests'
161 }
162 }
163
164 build_stage('Test: full', Settings.RunAllTests) {
165 dir (BuildDir) {
166 //Run the tests from the tests directory
167 sh 'make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes'
168 sh 'make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no '
169 }
170 }
171}
172
173def benchmark() {
174 build_stage('Benchmark', Settings.RunBenchmark) {
175 dir (BuildDir) {
176 //Append bench results
177 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
178 }
179 }
180}
181
182def build_doc() {
183 build_stage('Documentation', Settings.BuildDocumentation) {
184 dir ('doc/user') {
185 make_doc()
186 }
187
188 dir ('doc/refrat') {
189 make_doc()
190 }
191 }
192}
193
194def publish() {
195 build_stage('Publish', true) {
196
197 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
198
199 def groupCompile = new PlotGroup('Compilation', 'seconds', true)
200 def groupConcurrency = new PlotGroup('Concurrency', 'nanoseconds', false)
201
202 //Then publish the results
203 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , 'Compilation')
204 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile-diff' , groupCompile , 'Compilation Speed-Up')
205 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, 'Context Switching')
206 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch-diff', groupConcurrency, 'Context Switching Speed-Up')
207 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, 'Mutual Exclusion')
208 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex-diff' , groupConcurrency, 'Mutual Exclusion Speed-Up')
209 do_plot(Settings.RunBenchmark && Settings.Publish, 'signal' , groupConcurrency, 'Internal and External Scheduling')
210 do_plot(Settings.RunBenchmark && Settings.Publish, 'signal-diff' , groupConcurrency, 'Internal and External Scheduling Speed-Up')
211 }
212}
213
214//===========================================================================================================
215//Routine responsible of sending the email notification once the build is completed
216//===========================================================================================================
217def GitLogMessage() {
218 if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n"
219
220 sh "${SrcDir}/tools/PrettyGitLogs.sh ${SrcDir} ${BuildDir} ${Settings.GitOldRef} ${Settings.GitNewRef}"
221
222 def gitUpdate = readFile("${BuildDir}/GIT_UPDATE")
223 def gitLog = readFile("${BuildDir}/GIT_LOG")
224 def gitDiff = readFile("${BuildDir}/GIT_DIFF")
225
226 return """
227<pre>
228The branch ${env.BRANCH_NAME} has been updated.
229${gitUpdate}
230</pre>
231
232<p>Check console output at ${env.BUILD_URL} to view the results.</p>
233
234<p>- Status --------------------------------------------------------------</p>
235
236<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
237
238<p>- Log -----------------------------------------------------------------</p>
239
240<pre>
241${gitLog}
242</pre>
243
244<p>-----------------------------------------------------------------------</p>
245<pre>
246Summary of changes:
247${gitDiff}
248</pre>
249"""
250}
251
252//Standard build email notification
253def email(boolean log) {
254 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
255 //Configurations for email format
256 echo 'Notifying users of result'
257
258 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
259 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
260 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
261generated because of a git hooks/post-receive script following
262a ref change which was pushed to the C\u2200 repository.</p>
263""" + GitLogMessage()
264
265 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
266
267 if( Settings && !Settings.Silent ) {
268 //send email notification
269 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
270 } else {
271 echo "Would send email to: ${email_to}"
272 echo "With title: ${email_subject}"
273 echo "Content: \n${email_body}"
274 }
275}
276
277//===========================================================================================================
278// Helper classes/variables/routines
279//===========================================================================================================
280//Description of a compiler (Must be serializable since pipelines are persistent)
281class CC_Desc implements Serializable {
282 public String name
283 public String CXX
284 public String CC
285
286 CC_Desc(String name, String CXX, String CC) {
287 this.name = name
288 this.CXX = CXX
289 this.CC = CC
290 }
291}
292
293//Description of an architecture (Must be serializable since pipelines are persistent)
294class Arch_Desc implements Serializable {
295 public String name
296 public String flags
297 public String node
298
299 Arch_Desc(String name, String flags, String node) {
300 this.name = name
301 this.flags = flags
302 this.node = node
303 }
304}
305
306class BuildSettings implements Serializable {
307 public final CC_Desc Compiler
308 public final Arch_Desc Architecture
309 public final Boolean RunAllTests
310 public final Boolean RunBenchmark
311 public final Boolean BuildDocumentation
312 public final Boolean Publish
313 public final Boolean Silent
314 public final Boolean IsSandbox
315 public final String DescLong
316 public final String DescShort
317
318 public String GitNewRef
319 public String GitOldRef
320
321 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
322 switch( param.Compiler ) {
323 case 'gcc-6':
324 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6')
325 break
326 case 'gcc-5':
327 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5')
328 break
329 case 'gcc-4.9':
330 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9')
331 break
332 case 'clang':
333 this.Compiler = new CC_Desc('clang', 'clang++', 'gcc-6')
334 break
335 default :
336 error "Unhandled compiler : ${cc}"
337 }
338
339 switch( param.Architecture ) {
340 case 'x64':
341 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
342 break
343 case 'x86':
344 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
345 break
346 default :
347 error "Unhandled architecture : ${arch}"
348 }
349
350 this.IsSandbox = (branch == "jenkins-sandbox")
351 this.RunAllTests = param.RunAllTests
352 this.RunBenchmark = param.RunBenchmark
353 this.BuildDocumentation = param.BuildDocumentation
354 this.Publish = param.Publish
355 this.Silent = param.Silent
356
357 def full = param.RunAllTests ? " (Full)" : ""
358 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
359
360 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
361Architecture : ${ this.Architecture.name }
362Arc Flags : ${ this.Architecture.flags }
363Run All Tests : ${ this.RunAllTests.toString() }
364Run Benchmark : ${ this.RunBenchmark.toString() }
365Build Documentation : ${ this.BuildDocumentation.toString() }
366Publish : ${ this.Publish.toString() }
367Silent : ${ this.Silent.toString() }
368"""
369
370 this.GitNewRef = ''
371 this.GitOldRef = ''
372 }
373}
374
375class PlotGroup implements Serializable {
376 public String name
377 public String unit
378 public boolean log
379
380 PlotGroup(String name, String unit, boolean log) {
381 this.name = name
382 this.unit = unit
383 this.log = log
384 }
385}
386
387def prepare_build() {
388 // prepare the properties
389 properties ([ \
390 [$class: 'ParametersDefinitionProperty', \
391 parameterDefinitions: [ \
392 [$class: 'ChoiceParameterDefinition', \
393 description: 'Which compiler to use', \
394 name: 'Compiler', \
395 choices: 'gcc-6\ngcc-5\ngcc-4.9\nclang', \
396 defaultValue: 'gcc-6', \
397 ], \
398 [$class: 'ChoiceParameterDefinition', \
399 description: 'The target architecture', \
400 name: 'Architecture', \
401 choices: 'x64\nx86', \
402 defaultValue: 'x64', \
403 ], \
404 [$class: 'BooleanParameterDefinition', \
405 description: 'If false, only the quick test suite is ran', \
406 name: 'RunAllTests', \
407 defaultValue: false, \
408 ], \
409 [$class: 'BooleanParameterDefinition', \
410 description: 'If true, jenkins also runs benchmarks', \
411 name: 'RunBenchmark', \
412 defaultValue: false, \
413 ], \
414 [$class: 'BooleanParameterDefinition', \
415 description: 'If true, jenkins also builds documentation', \
416 name: 'BuildDocumentation', \
417 defaultValue: true, \
418 ], \
419 [$class: 'BooleanParameterDefinition', \
420 description: 'If true, jenkins also publishes results', \
421 name: 'Publish', \
422 defaultValue: false, \
423 ], \
424 [$class: 'BooleanParameterDefinition', \
425 description: 'If true, jenkins will not send emails', \
426 name: 'Silent', \
427 defaultValue: false, \
428 ], \
429 ],
430 ]])
431
432 // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer
433 checkout scm
434
435 final settings = new BuildSettings(params, env.BRANCH_NAME)
436
437 currentBuild.description = settings.DescShort
438 echo settings.DescLong
439
440 return settings
441}
442
443def build_stage(String name, boolean run, Closure block ) {
444 StageName = name
445 echo " -------- ${StageName} -------- "
446 if(run) {
447 stage(name, block)
448 } else {
449 stage(name) { Utils.markStageSkippedForConditional(STAGE_NAME) }
450 }
451}
452
453def make_doc() {
454 def err = null
455 try {
456 sh 'make clean > /dev/null'
457 sh 'make > /dev/null 2>&1'
458 }
459 catch (Exception caughtError) {
460 err = caughtError //rethrow error later
461 sh 'cat build/*.log'
462 }
463 finally {
464 if (err) throw err // Must re-throw exception to propagate error
465 }
466}
467
468def do_plot(boolean new_data, String file, PlotGroup group, String title) {
469
470 if(new_data) {
471 echo "Publishing new data"
472 }
473
474 def series = new_data ? [[
475 file: "${file}.csv",
476 exclusionValues: '',
477 displayTableFlag: false,
478 inclusionFlag: 'OFF',
479 url: ''
480 ]] : [];
481
482 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
483 dir("${BuildDir}/benchmark/") {
484 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
485 csvSeries: series,
486 group: "${group.name}",
487 title: "${title}",
488 style: 'lineSimple',
489 exclZero: false,
490 keepRecords: false,
491 logarithmic: group.log,
492 numBuilds: '120',
493 useDescr: true,
494 yaxis: group.unit,
495 yaxisMaximum: '',
496 yaxisMinimum: ''
497 }
498}
Note: See TracBrowser for help on using the repository browser.