source: Jenkinsfile@ 1c54958

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 1c54958 was bd50205, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Jenkins email must run inside node

  • Property mode set to 100644
File size: 15.1 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5//===========================================================================================================
6// Main loop of the compilation
7//===========================================================================================================
8
9// Globals
10BuildDir = null
11SrcDir = null
12Settings = null
13Tools = null
14
15// Local variables
16def err = null
17def log_needed = false
18
19currentBuild.result = "SUCCESS"
20
21try {
22 node {
23 //Wrap build to add timestamp to command line
24 wrap([$class: 'TimestamperBuildWrapper']) {
25 Settings = prepare_build()
26 }
27 }
28
29 node(Settings.Architecture.node) {
30 //Wrap build to add timestamp to command line
31 wrap([$class: 'TimestamperBuildWrapper']) {
32 BuildDir = pwd tmp: true
33 SrcDir = pwd tmp: false
34 currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
35
36 Tools.Clean()
37
38 Tools.Checkout()
39
40 build()
41
42 test()
43
44 benchmark()
45
46 build_doc()
47
48 publish()
49 }
50 }
51}
52
53//If an exception is caught we need to change the status and remember to
54//attach the build log to the email
55catch (Exception caughtError) {
56 // Store the result of the build log
57 currentBuild.result = "FAILURE"
58
59 // An error has occured, the build log is relevent
60 log_needed = true
61
62 // rethrow error later
63 err = caughtError
64
65 // print the error so it shows in the log
66 echo err.toString()
67}
68
69finally {
70 //Send email with final results if this is not a full build
71 email(log_needed)
72
73 echo 'Build Completed'
74
75 /* Must re-throw exception to propagate error */
76 if (err) {
77 throw err
78 }
79}
80//===========================================================================================================
81// Main compilation routines
82//===========================================================================================================
83def build() {
84 debug = true
85 release = Settings.RunAllTests || Settings.RunBenchmark
86 Tools.BuildStage('Build : configure', true) {
87 // Configure must be run inside the tree
88 dir (SrcDir) {
89 // Generate the necessary build files
90 sh './autogen.sh'
91 }
92
93 // Build outside of the src tree to ease cleaning
94 dir (BuildDir) {
95 //Configure the compilation (Output is not relevant)
96 //Use the current directory as the installation target so nothing escapes the sandbox
97 //Also specify the compiler by hand
98 targets=""
99 if( Settings.RunAllTests || Settings.RunBenchmark ) {
100 targets="--with-target-hosts='host:debug,host:nodebug'"
101 } else {
102 targets="--with-target-hosts='host:debug'"
103 }
104
105 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
106
107 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
108
109 // Configure libcfa
110 sh 'make -j 8 --no-print-directory configure-libcfa'
111 }
112 }
113
114 Tools.BuildStage('Build : cfa-cpp', true) {
115 // Build outside of the src tree to ease cleaning
116 dir (BuildDir) {
117 // Build driver
118 sh 'make -j 8 --no-print-directory -C driver'
119
120 // Build translator
121 sh 'make -j 8 --no-print-directory -C src'
122 }
123 }
124
125 Tools.BuildStage('Build : libcfa(debug)', debug) {
126 // Build outside of the src tree to ease cleaning
127 dir (BuildDir) {
128 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
129 }
130 }
131
132 Tools.BuildStage('Build : libcfa(nodebug)', release) {
133 // Build outside of the src tree to ease cleaning
134 dir (BuildDir) {
135 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
136 }
137 }
138
139 Tools.BuildStage('Build : install', true) {
140 // Build outside of the src tree to ease cleaning
141 dir (BuildDir) {
142 sh "make -j 8 --no-print-directory install"
143 }
144 }
145}
146
147def test() {
148 try {
149 // Print potential limits before testing
150 // in case jenkins messes with them
151 sh 'free -h'
152 sh 'ulimit -a'
153
154 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
155 dir (BuildDir) {
156 //Run the tests from the tests directory
157 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
158 }
159 }
160
161 Tools.BuildStage('Test: full', Settings.RunAllTests) {
162 dir (BuildDir) {
163 //Run the tests from the tests directory
164 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
165 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
166 }
167 }
168 }
169 catch (Exception err) {
170 echo "Archiving core dumps"
171 dir (BuildDir) {
172 def exists = fileExists 'tests/crashes'
173 if( exists ) {
174 sh """${SrcDir}/tools/jenkins/archive-gen.sh"""
175 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*,setup.sh", fingerprint: true
176 }
177 }
178 throw err
179 }
180}
181
182def benchmark() {
183 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
184 dir (BuildDir) {
185 //Append bench results
186 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
187 }
188 }
189}
190
191def build_doc() {
192 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
193 dir ('doc/user') {
194 make_doc()
195 }
196
197 dir ('doc/refrat') {
198 make_doc()
199 }
200 }
201}
202
203def publish() {
204 Tools.BuildStage('Publish', true) {
205
206 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
207
208 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
209 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
210
211 //Then publish the results
212 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
213 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
214 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
215 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
216 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
217 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
218 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
219 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
220 }
221}
222
223//===========================================================================================================
224//Routine responsible of sending the email notification once the build is completed
225//===========================================================================================================
226//Standard build email notification
227def email(boolean log) {
228 node {
229 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
230 //Configurations for email format
231 echo 'Notifying users of result'
232
233 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
234 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
235 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
236 generated because of a git hooks/post-receive script following
237 a ref change which was pushed to the C\u2200 repository.</p>
238 """ + Tools.GitLogMessage()
239
240 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
241
242 if( Settings && !Settings.Silent ) {
243 //send email notification
244 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
245 } else {
246 echo "Would send email to: ${email_to}"
247 echo "With title: ${email_subject}"
248 echo "Content: \n${email_body}"
249 }
250 }
251}
252
253//===========================================================================================================
254// Helper classes/variables/routines
255//===========================================================================================================
256//Description of a compiler (Must be serializable since pipelines are persistent)
257class CC_Desc implements Serializable {
258 public String name
259 public String CXX
260 public String CC
261 public String lto
262
263 CC_Desc(String name, String CXX, String CC, String lto) {
264 this.name = name
265 this.CXX = CXX
266 this.CC = CC
267 this.lto = lto
268 }
269}
270
271//Description of an architecture (Must be serializable since pipelines are persistent)
272class Arch_Desc implements Serializable {
273 public String name
274 public String flags
275 public String node
276
277 Arch_Desc(String name, String flags, String node) {
278 this.name = name
279 this.flags = flags
280 this.node = node
281 }
282}
283
284class BuildSettings implements Serializable {
285 public final CC_Desc Compiler
286 public final Arch_Desc Architecture
287 public final Boolean NewAST
288 public final Boolean RunAllTests
289 public final Boolean RunBenchmark
290 public final Boolean BuildDocumentation
291 public final Boolean Publish
292 public final Boolean Silent
293 public final Boolean IsSandbox
294 public final String DescLong
295 public final String DescShort
296
297 public String GitNewRef
298 public String GitOldRef
299
300 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
301 switch( param.Compiler ) {
302 case 'gcc-9':
303 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
304 break
305 case 'gcc-8':
306 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
307 break
308 case 'gcc-7':
309 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
310 break
311 case 'gcc-6':
312 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
313 break
314 case 'gcc-5':
315 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
316 break
317 case 'gcc-4.9':
318 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
319 break
320 case 'clang':
321 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
322 break
323 default :
324 error "Unhandled compiler : ${cc}"
325 }
326
327 switch( param.Architecture ) {
328 case 'x64':
329 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
330 break
331 case 'x86':
332 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
333 break
334 default :
335 error "Unhandled architecture : ${arch}"
336 }
337
338 this.IsSandbox = (branch == "jenkins-sandbox")
339 this.NewAST = param.NewAST
340 this.RunAllTests = param.RunAllTests
341 this.RunBenchmark = param.RunBenchmark
342 this.BuildDocumentation = param.BuildDocumentation
343 this.Publish = param.Publish
344 this.Silent = param.Silent
345
346 def full = param.RunAllTests ? " (Full)" : ""
347 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
348
349 final ast = this.NewAST ? "New AST" : "Old AST"
350 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
351AST Version : ${ ast.toString() }
352Architecture : ${ this.Architecture.name }
353Arc Flags : ${ this.Architecture.flags }
354Run All Tests : ${ this.RunAllTests.toString() }
355Run Benchmark : ${ this.RunBenchmark.toString() }
356Build Documentation : ${ this.BuildDocumentation.toString() }
357Publish : ${ this.Publish.toString() }
358Silent : ${ this.Silent.toString() }
359"""
360
361 this.GitNewRef = ''
362 this.GitOldRef = ''
363 }
364}
365
366class PlotGroup implements Serializable {
367 public String name
368 public String unit
369 public boolean log
370
371 PlotGroup(String name, String unit, boolean log) {
372 this.name = name
373 this.unit = unit
374 this.log = log
375 }
376}
377
378def prepare_build() {
379 // prepare the properties
380 properties ([ \
381 buildDiscarder(logRotator( \
382 artifactDaysToKeepStr: '', \
383 artifactNumToKeepStr: '', \
384 daysToKeepStr: '730', \
385 numToKeepStr: '1000' \
386 )), \
387 [$class: 'ParametersDefinitionProperty', \
388 parameterDefinitions: [ \
389 [$class: 'ChoiceParameterDefinition', \
390 description: 'Which compiler to use', \
391 name: 'Compiler', \
392 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
393 defaultValue: 'gcc-8', \
394 ], \
395 [$class: 'ChoiceParameterDefinition', \
396 description: 'The target architecture', \
397 name: 'Architecture', \
398 choices: 'x64\nx86', \
399 defaultValue: 'x64', \
400 ], \
401 [$class: 'BooleanParameterDefinition', \
402 description: 'If true, build compiler using new AST', \
403 name: 'NewAST', \
404 defaultValue: true, \
405 ], \
406 [$class: 'BooleanParameterDefinition', \
407 description: 'If false, only the quick test suite is ran', \
408 name: 'RunAllTests', \
409 defaultValue: false, \
410 ], \
411 [$class: 'BooleanParameterDefinition', \
412 description: 'If true, jenkins also runs benchmarks', \
413 name: 'RunBenchmark', \
414 defaultValue: false, \
415 ], \
416 [$class: 'BooleanParameterDefinition', \
417 description: 'If true, jenkins also builds documentation', \
418 name: 'BuildDocumentation', \
419 defaultValue: true, \
420 ], \
421 [$class: 'BooleanParameterDefinition', \
422 description: 'If true, jenkins also publishes results', \
423 name: 'Publish', \
424 defaultValue: false, \
425 ], \
426 [$class: 'BooleanParameterDefinition', \
427 description: 'If true, jenkins will not send emails', \
428 name: 'Silent', \
429 defaultValue: false, \
430 ], \
431 ],
432 ]])
433
434 // It's unfortunate but it looks like we need to checkout the entire repo just to get
435 // - the pretty git printer
436 // - Jenkins.tools
437 checkout scm
438
439 Tools = load "Jenkins/tools.groovy"
440
441 final settings = new BuildSettings(params, env.BRANCH_NAME)
442
443 currentBuild.description = settings.DescShort
444 echo settings.DescLong
445
446 return settings
447}
448
449def make_doc() {
450 def err = null
451 try {
452 sh 'make clean > /dev/null'
453 sh 'make > /dev/null 2>&1'
454 }
455 catch (Exception caughtError) {
456 err = caughtError //rethrow error later
457 sh 'cat build/*.log'
458 }
459 finally {
460 if (err) throw err // Must re-throw exception to propagate error
461 }
462}
463
464def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
465
466 if(new_data) {
467 echo "Publishing new data"
468 }
469
470 def series = new_data ? [[
471 file: "${file}.csv",
472 exclusionValues: '',
473 displayTableFlag: false,
474 inclusionFlag: 'OFF',
475 url: ''
476 ]] : [];
477
478 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
479 dir("${BuildDir}/benchmark/") {
480 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
481 csvSeries: series,
482 group: "${group.name}",
483 title: "${title}",
484 style: 'lineSimple',
485 exclZero: false,
486 keepRecords: false,
487 logarithmic: !relative && group.log,
488 numBuilds: '120',
489 useDescr: true,
490 yaxis: group.unit,
491 yaxisMaximum: '',
492 yaxisMinimum: ''
493 }
494}
Note: See TracBrowser for help on using the repository browser.