source: Jenkinsfile@ 1baf6ed

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 1baf6ed was 1baf6ed, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Added memory check in jenkins file

  • Property mode set to 100644
File size: 14.9 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5//===========================================================================================================
6// Main loop of the compilation
7//===========================================================================================================
8
9node('master') {
10 // Globals
11 BuildDir = pwd tmp: true
12 SrcDir = pwd tmp: false
13 Settings = null
14 Tools = null
15
16 // Local variables
17 def err = null
18 def log_needed = false
19
20 currentBuild.result = "SUCCESS"
21
22 try {
23 //Wrap build to add timestamp to command line
24 wrap([$class: 'TimestamperBuildWrapper']) {
25
26 Settings = prepare_build()
27
28 node(Settings.Architecture.node) {
29 BuildDir = pwd tmp: true
30 SrcDir = pwd tmp: false
31
32 Tools.Clean()
33
34 Tools.Checkout()
35
36 build()
37
38 test()
39
40 benchmark()
41
42 build_doc()
43
44 publish()
45 }
46
47 // Update the build directories when exiting the node
48 BuildDir = pwd tmp: true
49 SrcDir = pwd tmp: false
50 }
51 }
52
53 //If an exception is caught we need to change the status and remember to
54 //attach the build log to the email
55 catch (Exception caughtError) {
56 // Store the result of the build log
57 currentBuild.result = "FAILURE"
58
59 // An error has occured, the build log is relevent
60 log_needed = true
61
62 // rethrow error later
63 err = caughtError
64
65 // print the error so it shows in the log
66 echo err.toString()
67 }
68
69 finally {
70 //Send email with final results if this is not a full build
71 email(log_needed)
72
73 echo 'Build Completed'
74
75 /* Must re-throw exception to propagate error */
76 if (err) {
77 throw err
78 }
79 }
80}
81//===========================================================================================================
82// Main compilation routines
83//===========================================================================================================
84def build() {
85 debug = true
86 release = Settings.RunAllTests || Settings.RunBenchmark
87 Tools.BuildStage('Build : configure', true) {
88 // Configure must be run inside the tree
89 dir (SrcDir) {
90 // Generate the necessary build files
91 sh './autogen.sh'
92 }
93
94 // Build outside of the src tree to ease cleaning
95 dir (BuildDir) {
96 //Configure the compilation (Output is not relevant)
97 //Use the current directory as the installation target so nothing escapes the sandbox
98 //Also specify the compiler by hand
99 targets=""
100 if( Settings.RunAllTests || Settings.RunBenchmark ) {
101 targets="--with-target-hosts='host:debug,host:nodebug'"
102 } else {
103 targets="--with-target-hosts='host:debug'"
104 }
105
106 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
107
108 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
109
110 // Configure libcfa
111 sh 'make -j 8 --no-print-directory configure-libcfa'
112 }
113 }
114
115 Tools.BuildStage('Build : cfa-cpp', true) {
116 // Build outside of the src tree to ease cleaning
117 dir (BuildDir) {
118 // Build driver
119 sh 'make -j 8 --no-print-directory -C driver'
120
121 // Build translator
122 sh 'make -j 8 --no-print-directory -C src'
123 }
124 }
125
126 Tools.BuildStage('Build : libcfa(debug)', debug) {
127 // Build outside of the src tree to ease cleaning
128 dir (BuildDir) {
129 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
130 }
131 }
132
133 Tools.BuildStage('Build : libcfa(nodebug)', release) {
134 // Build outside of the src tree to ease cleaning
135 dir (BuildDir) {
136 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
137 }
138 }
139
140 Tools.BuildStage('Build : install', true) {
141 // Build outside of the src tree to ease cleaning
142 dir (BuildDir) {
143 sh "make -j 8 --no-print-directory install"
144 }
145 }
146}
147
148def test() {
149 try {
150 // Print potential limits before testing
151 // in case jenkins messes with them
152 sh 'ulimit -a'
153
154 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
155 dir (BuildDir) {
156 //Run the tests from the tests directory
157 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
158 }
159 }
160
161 Tools.BuildStage('Test: full', Settings.RunAllTests) {
162 dir (BuildDir) {
163 //Run the tests from the tests directory
164 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
165 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
166 }
167 }
168 }
169 catch (Exception err) {
170 echo "Archiving core dumps"
171 dir (BuildDir) {
172 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
173 }
174 throw err
175 }
176}
177
178def benchmark() {
179 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
180 dir (BuildDir) {
181 //Append bench results
182 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
183 }
184 }
185}
186
187def build_doc() {
188 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
189 dir ('doc/user') {
190 make_doc()
191 }
192
193 dir ('doc/refrat') {
194 make_doc()
195 }
196 }
197}
198
199def publish() {
200 Tools.BuildStage('Publish', true) {
201
202 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
203
204 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
205 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
206
207 //Then publish the results
208 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
209 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
210 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
211 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
212 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
213 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
214 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
215 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
216 }
217}
218
219//===========================================================================================================
220//Routine responsible of sending the email notification once the build is completed
221//===========================================================================================================
222//Standard build email notification
223def email(boolean log) {
224 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
225 //Configurations for email format
226 echo 'Notifying users of result'
227
228 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
229 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
230 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
231generated because of a git hooks/post-receive script following
232a ref change which was pushed to the C\u2200 repository.</p>
233""" + Tools.GitLogMessage()
234
235 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
236
237 if( Settings && !Settings.Silent ) {
238 //send email notification
239 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
240 } else {
241 echo "Would send email to: ${email_to}"
242 echo "With title: ${email_subject}"
243 echo "Content: \n${email_body}"
244 }
245}
246
247//===========================================================================================================
248// Helper classes/variables/routines
249//===========================================================================================================
250//Description of a compiler (Must be serializable since pipelines are persistent)
251class CC_Desc implements Serializable {
252 public String name
253 public String CXX
254 public String CC
255 public String lto
256
257 CC_Desc(String name, String CXX, String CC, String lto) {
258 this.name = name
259 this.CXX = CXX
260 this.CC = CC
261 this.lto = lto
262 }
263}
264
265//Description of an architecture (Must be serializable since pipelines are persistent)
266class Arch_Desc implements Serializable {
267 public String name
268 public String flags
269 public String node
270
271 Arch_Desc(String name, String flags, String node) {
272 this.name = name
273 this.flags = flags
274 this.node = node
275 }
276}
277
278class BuildSettings implements Serializable {
279 public final CC_Desc Compiler
280 public final Arch_Desc Architecture
281 public final Boolean NewAST
282 public final Boolean RunAllTests
283 public final Boolean RunBenchmark
284 public final Boolean BuildDocumentation
285 public final Boolean Publish
286 public final Boolean Silent
287 public final Boolean IsSandbox
288 public final String DescLong
289 public final String DescShort
290
291 public String GitNewRef
292 public String GitOldRef
293
294 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
295 switch( param.Compiler ) {
296 case 'gcc-9':
297 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
298 break
299 case 'gcc-8':
300 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
301 break
302 case 'gcc-7':
303 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
304 break
305 case 'gcc-6':
306 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
307 break
308 case 'gcc-5':
309 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
310 break
311 case 'gcc-4.9':
312 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
313 break
314 case 'clang':
315 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
316 break
317 default :
318 error "Unhandled compiler : ${cc}"
319 }
320
321 switch( param.Architecture ) {
322 case 'x64':
323 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
324 break
325 case 'x86':
326 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
327 break
328 default :
329 error "Unhandled architecture : ${arch}"
330 }
331
332 this.IsSandbox = (branch == "jenkins-sandbox")
333 this.NewAST = param.NewAST
334 this.RunAllTests = param.RunAllTests
335 this.RunBenchmark = param.RunBenchmark
336 this.BuildDocumentation = param.BuildDocumentation
337 this.Publish = param.Publish
338 this.Silent = param.Silent
339
340 def full = param.RunAllTests ? " (Full)" : ""
341 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
342
343 final ast = this.NewAST ? "New AST" : "Old AST"
344 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
345AST Version : ${ ast.toString() }
346Architecture : ${ this.Architecture.name }
347Arc Flags : ${ this.Architecture.flags }
348Run All Tests : ${ this.RunAllTests.toString() }
349Run Benchmark : ${ this.RunBenchmark.toString() }
350Build Documentation : ${ this.BuildDocumentation.toString() }
351Publish : ${ this.Publish.toString() }
352Silent : ${ this.Silent.toString() }
353"""
354
355 this.GitNewRef = ''
356 this.GitOldRef = ''
357 }
358}
359
360class PlotGroup implements Serializable {
361 public String name
362 public String unit
363 public boolean log
364
365 PlotGroup(String name, String unit, boolean log) {
366 this.name = name
367 this.unit = unit
368 this.log = log
369 }
370}
371
372def prepare_build() {
373 // prepare the properties
374 properties ([ \
375 buildDiscarder(logRotator( \
376 artifactDaysToKeepStr: '', \
377 artifactNumToKeepStr: '', \
378 daysToKeepStr: '730', \
379 numToKeepStr: '1000' \
380 )), \
381 [$class: 'ParametersDefinitionProperty', \
382 parameterDefinitions: [ \
383 [$class: 'ChoiceParameterDefinition', \
384 description: 'Which compiler to use', \
385 name: 'Compiler', \
386 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
387 defaultValue: 'gcc-8', \
388 ], \
389 [$class: 'ChoiceParameterDefinition', \
390 description: 'The target architecture', \
391 name: 'Architecture', \
392 choices: 'x64\nx86', \
393 defaultValue: 'x64', \
394 ], \
395 [$class: 'BooleanParameterDefinition', \
396 description: 'If true, build compiler using new AST', \
397 name: 'NewAST', \
398 defaultValue: true, \
399 ], \
400 [$class: 'BooleanParameterDefinition', \
401 description: 'If false, only the quick test suite is ran', \
402 name: 'RunAllTests', \
403 defaultValue: false, \
404 ], \
405 [$class: 'BooleanParameterDefinition', \
406 description: 'If true, jenkins also runs benchmarks', \
407 name: 'RunBenchmark', \
408 defaultValue: false, \
409 ], \
410 [$class: 'BooleanParameterDefinition', \
411 description: 'If true, jenkins also builds documentation', \
412 name: 'BuildDocumentation', \
413 defaultValue: true, \
414 ], \
415 [$class: 'BooleanParameterDefinition', \
416 description: 'If true, jenkins also publishes results', \
417 name: 'Publish', \
418 defaultValue: false, \
419 ], \
420 [$class: 'BooleanParameterDefinition', \
421 description: 'If true, jenkins will not send emails', \
422 name: 'Silent', \
423 defaultValue: false, \
424 ], \
425 ],
426 ]])
427
428 // It's unfortunate but it looks like we need to checkout the entire repo just to get
429 // - the pretty git printer
430 // - Jenkins.tools
431 checkout scm
432
433 Tools = load "Jenkins/tools.groovy"
434
435 final settings = new BuildSettings(params, env.BRANCH_NAME)
436
437 currentBuild.description = settings.DescShort
438 echo settings.DescLong
439
440 return settings
441}
442
443def make_doc() {
444 def err = null
445 try {
446 sh 'make clean > /dev/null'
447 sh 'make > /dev/null 2>&1'
448 }
449 catch (Exception caughtError) {
450 err = caughtError //rethrow error later
451 sh 'cat build/*.log'
452 }
453 finally {
454 if (err) throw err // Must re-throw exception to propagate error
455 }
456}
457
458def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
459
460 if(new_data) {
461 echo "Publishing new data"
462 }
463
464 def series = new_data ? [[
465 file: "${file}.csv",
466 exclusionValues: '',
467 displayTableFlag: false,
468 inclusionFlag: 'OFF',
469 url: ''
470 ]] : [];
471
472 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
473 dir("${BuildDir}/benchmark/") {
474 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
475 csvSeries: series,
476 group: "${group.name}",
477 title: "${title}",
478 style: 'lineSimple',
479 exclZero: false,
480 keepRecords: false,
481 logarithmic: !relative && group.log,
482 numBuilds: '120',
483 useDescr: true,
484 yaxis: group.unit,
485 yaxisMaximum: '',
486 yaxisMinimum: ''
487 }
488}
Note: See TracBrowser for help on using the repository browser.