1 | #!groovy |
---|
2 | |
---|
3 | import groovy.transform.Field |
---|
4 | |
---|
5 | // For skipping stages |
---|
6 | import org.jenkinsci.plugins.pipeline.modeldefinition.Utils |
---|
7 | |
---|
8 | //=========================================================================================================== |
---|
9 | // Main loop of the compilation |
---|
10 | //=========================================================================================================== |
---|
11 | |
---|
12 | node('master') { |
---|
13 | // Globals |
---|
14 | BuildDir = pwd tmp: true |
---|
15 | SrcDir = pwd tmp: false |
---|
16 | Settings = null |
---|
17 | StageName = '' |
---|
18 | |
---|
19 | // Local variables |
---|
20 | def err = null |
---|
21 | def log_needed = false |
---|
22 | |
---|
23 | currentBuild.result = "SUCCESS" |
---|
24 | |
---|
25 | try { |
---|
26 | //Wrap build to add timestamp to command line |
---|
27 | wrap([$class: 'TimestamperBuildWrapper']) { |
---|
28 | |
---|
29 | Settings = prepare_build() |
---|
30 | |
---|
31 | node(Settings.Architecture.node) { |
---|
32 | BuildDir = pwd tmp: true |
---|
33 | SrcDir = pwd tmp: false |
---|
34 | |
---|
35 | clean() |
---|
36 | |
---|
37 | checkout() |
---|
38 | |
---|
39 | build() |
---|
40 | |
---|
41 | test() |
---|
42 | |
---|
43 | benchmark() |
---|
44 | |
---|
45 | build_doc() |
---|
46 | |
---|
47 | publish() |
---|
48 | } |
---|
49 | |
---|
50 | // Update the build directories when exiting the node |
---|
51 | BuildDir = pwd tmp: true |
---|
52 | SrcDir = pwd tmp: false |
---|
53 | } |
---|
54 | } |
---|
55 | |
---|
56 | //If an exception is caught we need to change the status and remember to |
---|
57 | //attach the build log to the email |
---|
58 | catch (Exception caughtError) { |
---|
59 | //rethrow error later |
---|
60 | err = caughtError |
---|
61 | |
---|
62 | echo err.toString() |
---|
63 | |
---|
64 | //An error has occured, the build log is relevent |
---|
65 | log_needed = true |
---|
66 | |
---|
67 | //Store the result of the build log |
---|
68 | currentBuild.result = "${StageName} FAILURE".trim() |
---|
69 | } |
---|
70 | |
---|
71 | finally { |
---|
72 | //Send email with final results if this is not a full build |
---|
73 | email(log_needed) |
---|
74 | |
---|
75 | echo 'Build Completed' |
---|
76 | |
---|
77 | /* Must re-throw exception to propagate error */ |
---|
78 | if (err) { |
---|
79 | throw err |
---|
80 | } |
---|
81 | } |
---|
82 | } |
---|
83 | //=========================================================================================================== |
---|
84 | // Main compilation routines |
---|
85 | //=========================================================================================================== |
---|
86 | def clean() { |
---|
87 | build_stage('Cleanup', true) { |
---|
88 | // clean the build by wipping the build directory |
---|
89 | dir(BuildDir) { |
---|
90 | deleteDir() |
---|
91 | } |
---|
92 | } |
---|
93 | } |
---|
94 | |
---|
95 | //Compilation script is done here but environnement set-up and error handling is done in main loop |
---|
96 | def checkout() { |
---|
97 | build_stage('Checkout', true) { |
---|
98 | //checkout the source code and clean the repo |
---|
99 | final scmVars = checkout scm |
---|
100 | Settings.GitNewRef = scmVars.GIT_COMMIT |
---|
101 | Settings.GitOldRef = scmVars.GIT_PREVIOUS_COMMIT |
---|
102 | |
---|
103 | echo GitLogMessage() |
---|
104 | } |
---|
105 | } |
---|
106 | |
---|
107 | def build() { |
---|
108 | debug = true |
---|
109 | release = Settings.RunAllTests || Settings.RunBenchmark |
---|
110 | build_stage('Build : configure', true) { |
---|
111 | // Build outside of the src tree to ease cleaning |
---|
112 | dir (BuildDir) { |
---|
113 | //Configure the conpilation (Output is not relevant) |
---|
114 | //Use the current directory as the installation target so nothing escapes the sandbox |
---|
115 | //Also specify the compiler by hand |
---|
116 | targets="" |
---|
117 | if( Settings.RunAllTests || Settings.RunBenchmark ) { |
---|
118 | targets="--with-target-hosts='host:debug,host:nodebug'" |
---|
119 | } else { |
---|
120 | targets="--with-target-hosts='host:debug'" |
---|
121 | } |
---|
122 | |
---|
123 | sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} ${targets} --quiet" |
---|
124 | |
---|
125 | // Configure libcfa |
---|
126 | sh 'make -j 8 --no-print-directory configure-libcfa' |
---|
127 | } |
---|
128 | } |
---|
129 | |
---|
130 | build_stage('Build : cfa-cpp', true) { |
---|
131 | // Build outside of the src tree to ease cleaning |
---|
132 | dir (BuildDir) { |
---|
133 | // Build driver |
---|
134 | sh 'make -j 8 --no-print-directory -C driver' |
---|
135 | |
---|
136 | // Build translator |
---|
137 | sh 'make -j 8 --no-print-directory -C src' |
---|
138 | } |
---|
139 | } |
---|
140 | |
---|
141 | build_stage('Build : libcfa(debug)', debug) { |
---|
142 | // Build outside of the src tree to ease cleaning |
---|
143 | dir (BuildDir) { |
---|
144 | sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug" |
---|
145 | } |
---|
146 | } |
---|
147 | |
---|
148 | build_stage('Build : libcfa(nodebug)', release) { |
---|
149 | // Build outside of the src tree to ease cleaning |
---|
150 | dir (BuildDir) { |
---|
151 | sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug" |
---|
152 | } |
---|
153 | } |
---|
154 | } |
---|
155 | |
---|
156 | def test() { |
---|
157 | try { |
---|
158 | build_stage('Test: short', !Settings.RunAllTests) { |
---|
159 | dir (BuildDir) { |
---|
160 | //Run the tests from the tests directory |
---|
161 | sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short" |
---|
162 | } |
---|
163 | } |
---|
164 | |
---|
165 | build_stage('Test: full', Settings.RunAllTests) { |
---|
166 | dir (BuildDir) { |
---|
167 | //Run the tests from the tests directory |
---|
168 | sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug""" |
---|
169 | sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug""" |
---|
170 | } |
---|
171 | } |
---|
172 | } |
---|
173 | catch (Exception err) { |
---|
174 | echo "Archiving core dumps" |
---|
175 | dir (BuildDir) { |
---|
176 | archiveArtifacts artifacts: "tests/crashes/**/*", fingerprint: true |
---|
177 | } |
---|
178 | throw err |
---|
179 | } |
---|
180 | } |
---|
181 | |
---|
182 | def benchmark() { |
---|
183 | build_stage('Benchmark', Settings.RunBenchmark) { |
---|
184 | dir (BuildDir) { |
---|
185 | //Append bench results |
---|
186 | sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}" |
---|
187 | } |
---|
188 | } |
---|
189 | } |
---|
190 | |
---|
191 | def build_doc() { |
---|
192 | build_stage('Documentation', Settings.BuildDocumentation) { |
---|
193 | dir ('doc/user') { |
---|
194 | make_doc() |
---|
195 | } |
---|
196 | |
---|
197 | dir ('doc/refrat') { |
---|
198 | make_doc() |
---|
199 | } |
---|
200 | } |
---|
201 | } |
---|
202 | |
---|
203 | def publish() { |
---|
204 | build_stage('Publish', true) { |
---|
205 | |
---|
206 | if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' } |
---|
207 | |
---|
208 | def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true) |
---|
209 | def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false) |
---|
210 | |
---|
211 | //Then publish the results |
---|
212 | do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation') |
---|
213 | do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)') |
---|
214 | do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching') |
---|
215 | do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff', groupConcurrency, true , 'Context Switching (relative)') |
---|
216 | do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion') |
---|
217 | do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)') |
---|
218 | do_plot(Settings.RunBenchmark && Settings.Publish, 'signal' , groupConcurrency, false, 'Internal and External Scheduling') |
---|
219 | do_plot(Settings.RunBenchmark && Settings.Publish, 'signal.diff' , groupConcurrency, true , 'Internal and External Scheduling (relative)') |
---|
220 | } |
---|
221 | } |
---|
222 | |
---|
223 | //=========================================================================================================== |
---|
224 | //Routine responsible of sending the email notification once the build is completed |
---|
225 | //=========================================================================================================== |
---|
226 | def GitLogMessage() { |
---|
227 | if (!Settings || !Settings.GitOldRef || !Settings.GitNewRef) return "\nERROR retrieveing git information!\n" |
---|
228 | |
---|
229 | def oldRef = Settings.GitOldRef |
---|
230 | def newRef = Settings.GitNewRef |
---|
231 | |
---|
232 | def revText = sh(returnStdout: true, script: "git rev-list ${oldRef}..${newRef}").trim() |
---|
233 | def revList = [] |
---|
234 | revText.eachLine { |
---|
235 | revList += it |
---|
236 | } |
---|
237 | |
---|
238 | def gitUpdate = "" |
---|
239 | revList.each { rev -> |
---|
240 | def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim() |
---|
241 | gitUpdate = gitUpdate + " via ${rev} (${type})" |
---|
242 | } |
---|
243 | |
---|
244 | def rev = oldRef |
---|
245 | def type = sh(returnStdout: true, script: "git cat-file -t ${rev}").trim() |
---|
246 | gitUpdate = gitUpdate + " from ${rev} (${type})" |
---|
247 | |
---|
248 | def gitLog = sh(returnStdout: true, script: "git rev-list --format=short ${oldRef}...${newRef}").trim() |
---|
249 | |
---|
250 | def gitDiff = sh(returnStdout: true, script: "git diff --stat --color ${newRef} ${oldRef}").trim() |
---|
251 | gitDiff = gitDiff.replace('[32m', '<span style="color: #00AA00;">') |
---|
252 | gitDiff = gitDiff.replace('[31m', '<span style="color: #AA0000;">') |
---|
253 | gitDiff = gitDiff.replace('[m', '</span>') |
---|
254 | |
---|
255 | return """ |
---|
256 | <pre> |
---|
257 | The branch ${env.BRANCH_NAME} has been updated. |
---|
258 | ${gitUpdate} |
---|
259 | </pre> |
---|
260 | |
---|
261 | <p>Check console output at ${env.BUILD_URL} to view the results.</p> |
---|
262 | |
---|
263 | <p>- Status --------------------------------------------------------------</p> |
---|
264 | |
---|
265 | <p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p> |
---|
266 | |
---|
267 | <p>- Log -----------------------------------------------------------------</p> |
---|
268 | |
---|
269 | <pre> |
---|
270 | ${gitLog} |
---|
271 | </pre> |
---|
272 | |
---|
273 | <p>-----------------------------------------------------------------------</p> |
---|
274 | <pre> |
---|
275 | Summary of changes: |
---|
276 | ${gitDiff} |
---|
277 | </pre> |
---|
278 | """ |
---|
279 | } |
---|
280 | |
---|
281 | //Standard build email notification |
---|
282 | def email(boolean log) { |
---|
283 | //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line |
---|
284 | //Configurations for email format |
---|
285 | echo 'Notifying users of result' |
---|
286 | |
---|
287 | def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase() |
---|
288 | def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}" |
---|
289 | def email_body = """<p>This is an automated email from the Jenkins build machine. It was |
---|
290 | generated because of a git hooks/post-receive script following |
---|
291 | a ref change which was pushed to the C\u2200 repository.</p> |
---|
292 | """ + GitLogMessage() |
---|
293 | |
---|
294 | def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca" |
---|
295 | |
---|
296 | if( Settings && !Settings.Silent ) { |
---|
297 | //send email notification |
---|
298 | emailext body: email_body, subject: email_subject, to: email_to, attachLog: log |
---|
299 | } else { |
---|
300 | echo "Would send email to: ${email_to}" |
---|
301 | echo "With title: ${email_subject}" |
---|
302 | echo "Content: \n${email_body}" |
---|
303 | } |
---|
304 | } |
---|
305 | |
---|
306 | //=========================================================================================================== |
---|
307 | // Helper classes/variables/routines |
---|
308 | //=========================================================================================================== |
---|
309 | //Description of a compiler (Must be serializable since pipelines are persistent) |
---|
310 | class CC_Desc implements Serializable { |
---|
311 | public String name |
---|
312 | public String CXX |
---|
313 | public String CC |
---|
314 | |
---|
315 | CC_Desc(String name, String CXX, String CC) { |
---|
316 | this.name = name |
---|
317 | this.CXX = CXX |
---|
318 | this.CC = CC |
---|
319 | } |
---|
320 | } |
---|
321 | |
---|
322 | //Description of an architecture (Must be serializable since pipelines are persistent) |
---|
323 | class Arch_Desc implements Serializable { |
---|
324 | public String name |
---|
325 | public String flags |
---|
326 | public String node |
---|
327 | |
---|
328 | Arch_Desc(String name, String flags, String node) { |
---|
329 | this.name = name |
---|
330 | this.flags = flags |
---|
331 | this.node = node |
---|
332 | } |
---|
333 | } |
---|
334 | |
---|
335 | class BuildSettings implements Serializable { |
---|
336 | public final CC_Desc Compiler |
---|
337 | public final Arch_Desc Architecture |
---|
338 | public final Boolean RunAllTests |
---|
339 | public final Boolean RunBenchmark |
---|
340 | public final Boolean BuildDocumentation |
---|
341 | public final Boolean Publish |
---|
342 | public final Boolean Silent |
---|
343 | public final Boolean IsSandbox |
---|
344 | public final String DescLong |
---|
345 | public final String DescShort |
---|
346 | |
---|
347 | public String GitNewRef |
---|
348 | public String GitOldRef |
---|
349 | |
---|
350 | BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) { |
---|
351 | switch( param.Compiler ) { |
---|
352 | case 'gcc-6': |
---|
353 | this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6') |
---|
354 | break |
---|
355 | case 'gcc-5': |
---|
356 | this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5') |
---|
357 | break |
---|
358 | case 'gcc-4.9': |
---|
359 | this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9') |
---|
360 | break |
---|
361 | case 'clang': |
---|
362 | this.Compiler = new CC_Desc('clang', 'clang++', 'gcc-6') |
---|
363 | break |
---|
364 | default : |
---|
365 | error "Unhandled compiler : ${cc}" |
---|
366 | } |
---|
367 | |
---|
368 | switch( param.Architecture ) { |
---|
369 | case 'x64': |
---|
370 | this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64') |
---|
371 | break |
---|
372 | case 'x86': |
---|
373 | this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86') |
---|
374 | break |
---|
375 | default : |
---|
376 | error "Unhandled architecture : ${arch}" |
---|
377 | } |
---|
378 | |
---|
379 | this.IsSandbox = (branch == "jenkins-sandbox") |
---|
380 | this.RunAllTests = param.RunAllTests |
---|
381 | this.RunBenchmark = param.RunBenchmark |
---|
382 | this.BuildDocumentation = param.BuildDocumentation |
---|
383 | this.Publish = param.Publish |
---|
384 | this.Silent = param.Silent |
---|
385 | |
---|
386 | def full = param.RunAllTests ? " (Full)" : "" |
---|
387 | this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}" |
---|
388 | |
---|
389 | this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC }) |
---|
390 | Architecture : ${ this.Architecture.name } |
---|
391 | Arc Flags : ${ this.Architecture.flags } |
---|
392 | Run All Tests : ${ this.RunAllTests.toString() } |
---|
393 | Run Benchmark : ${ this.RunBenchmark.toString() } |
---|
394 | Build Documentation : ${ this.BuildDocumentation.toString() } |
---|
395 | Publish : ${ this.Publish.toString() } |
---|
396 | Silent : ${ this.Silent.toString() } |
---|
397 | """ |
---|
398 | |
---|
399 | this.GitNewRef = '' |
---|
400 | this.GitOldRef = '' |
---|
401 | } |
---|
402 | } |
---|
403 | |
---|
404 | class PlotGroup implements Serializable { |
---|
405 | public String name |
---|
406 | public String unit |
---|
407 | public boolean log |
---|
408 | |
---|
409 | PlotGroup(String name, String unit, boolean log) { |
---|
410 | this.name = name |
---|
411 | this.unit = unit |
---|
412 | this.log = log |
---|
413 | } |
---|
414 | } |
---|
415 | |
---|
416 | def prepare_build() { |
---|
417 | // prepare the properties |
---|
418 | properties ([ \ |
---|
419 | [$class: 'ParametersDefinitionProperty', \ |
---|
420 | parameterDefinitions: [ \ |
---|
421 | [$class: 'ChoiceParameterDefinition', \ |
---|
422 | description: 'Which compiler to use', \ |
---|
423 | name: 'Compiler', \ |
---|
424 | choices: 'gcc-6\ngcc-5\ngcc-4.9\nclang', \ |
---|
425 | defaultValue: 'gcc-6', \ |
---|
426 | ], \ |
---|
427 | [$class: 'ChoiceParameterDefinition', \ |
---|
428 | description: 'The target architecture', \ |
---|
429 | name: 'Architecture', \ |
---|
430 | choices: 'x64\nx86', \ |
---|
431 | defaultValue: 'x64', \ |
---|
432 | ], \ |
---|
433 | [$class: 'BooleanParameterDefinition', \ |
---|
434 | description: 'If false, only the quick test suite is ran', \ |
---|
435 | name: 'RunAllTests', \ |
---|
436 | defaultValue: false, \ |
---|
437 | ], \ |
---|
438 | [$class: 'BooleanParameterDefinition', \ |
---|
439 | description: 'If true, jenkins also runs benchmarks', \ |
---|
440 | name: 'RunBenchmark', \ |
---|
441 | defaultValue: false, \ |
---|
442 | ], \ |
---|
443 | [$class: 'BooleanParameterDefinition', \ |
---|
444 | description: 'If true, jenkins also builds documentation', \ |
---|
445 | name: 'BuildDocumentation', \ |
---|
446 | defaultValue: true, \ |
---|
447 | ], \ |
---|
448 | [$class: 'BooleanParameterDefinition', \ |
---|
449 | description: 'If true, jenkins also publishes results', \ |
---|
450 | name: 'Publish', \ |
---|
451 | defaultValue: false, \ |
---|
452 | ], \ |
---|
453 | [$class: 'BooleanParameterDefinition', \ |
---|
454 | description: 'If true, jenkins will not send emails', \ |
---|
455 | name: 'Silent', \ |
---|
456 | defaultValue: false, \ |
---|
457 | ], \ |
---|
458 | ], |
---|
459 | ]]) |
---|
460 | |
---|
461 | // It's unfortunate but it looks like we need to checkout the entire repo just to get the pretty git printer |
---|
462 | checkout scm |
---|
463 | |
---|
464 | final settings = new BuildSettings(params, env.BRANCH_NAME) |
---|
465 | |
---|
466 | currentBuild.description = settings.DescShort |
---|
467 | echo settings.DescLong |
---|
468 | |
---|
469 | return settings |
---|
470 | } |
---|
471 | |
---|
472 | def build_stage(String name, boolean run, Closure block ) { |
---|
473 | StageName = name |
---|
474 | echo " -------- ${StageName} -------- " |
---|
475 | if(run) { |
---|
476 | stage(name, block) |
---|
477 | } else { |
---|
478 | stage(name) { Utils.markStageSkippedForConditional(STAGE_NAME) } |
---|
479 | } |
---|
480 | } |
---|
481 | |
---|
482 | def make_doc() { |
---|
483 | def err = null |
---|
484 | try { |
---|
485 | sh 'make clean > /dev/null' |
---|
486 | sh 'make > /dev/null 2>&1' |
---|
487 | } |
---|
488 | catch (Exception caughtError) { |
---|
489 | err = caughtError //rethrow error later |
---|
490 | sh 'cat build/*.log' |
---|
491 | } |
---|
492 | finally { |
---|
493 | if (err) throw err // Must re-throw exception to propagate error |
---|
494 | } |
---|
495 | } |
---|
496 | |
---|
497 | def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) { |
---|
498 | |
---|
499 | if(new_data) { |
---|
500 | echo "Publishing new data" |
---|
501 | } |
---|
502 | |
---|
503 | def series = new_data ? [[ |
---|
504 | file: "${file}.csv", |
---|
505 | exclusionValues: '', |
---|
506 | displayTableFlag: false, |
---|
507 | inclusionFlag: 'OFF', |
---|
508 | url: '' |
---|
509 | ]] : []; |
---|
510 | |
---|
511 | echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}" |
---|
512 | dir("${BuildDir}/benchmark/") { |
---|
513 | plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv", |
---|
514 | csvSeries: series, |
---|
515 | group: "${group.name}", |
---|
516 | title: "${title}", |
---|
517 | style: 'lineSimple', |
---|
518 | exclZero: false, |
---|
519 | keepRecords: false, |
---|
520 | logarithmic: !relative && group.log, |
---|
521 | numBuilds: '120', |
---|
522 | useDescr: true, |
---|
523 | yaxis: group.unit, |
---|
524 | yaxisMaximum: '', |
---|
525 | yaxisMinimum: '' |
---|
526 | } |
---|
527 | } |
---|