source: Jenkinsfile@ 361bf01

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 361bf01 was 8e58264, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

First attempt at full distribute.
Fixed some typos.

  • Property mode set to 100644
File size: 14.8 KB
Line 
1#!groovy
2
3import groovy.transform.Field
4
5//===========================================================================================================
6// Main loop of the compilation
7//===========================================================================================================
8
9node('master') {
10 // Globals
11 BuildDir = pwd tmp: true
12 SrcDir = pwd tmp: false
13 Settings = null
14 Tools = null
15
16 // Local variables
17 def err = null
18 def log_needed = false
19
20 currentBuild.result = "SUCCESS"
21
22 try {
23 //Wrap build to add timestamp to command line
24 wrap([$class: 'TimestamperBuildWrapper']) {
25
26 Settings = prepare_build()
27
28 node(Settings.Architecture.node) {
29 BuildDir = pwd tmp: true
30 SrcDir = pwd tmp: false
31
32 Tools.Clean()
33
34 Tools.Checkout()
35
36 build()
37
38 test()
39
40 benchmark()
41
42 build_doc()
43
44 publish()
45 }
46
47 // Update the build directories when exiting the node
48 BuildDir = pwd tmp: true
49 SrcDir = pwd tmp: false
50 }
51 }
52
53 //If an exception is caught we need to change the status and remember to
54 //attach the build log to the email
55 catch (Exception caughtError) {
56 //rethrow error later
57 err = caughtError
58
59 echo err.toString()
60
61 //An error has occured, the build log is relevent
62 log_needed = true
63
64 //Store the result of the build log
65 currentBuild.result = "${tools.StageName} FAILURE".trim()
66 }
67
68 finally {
69 //Send email with final results if this is not a full build
70 email(log_needed)
71
72 echo 'Build Completed'
73
74 /* Must re-throw exception to propagate error */
75 if (err) {
76 throw err
77 }
78 }
79}
80//===========================================================================================================
81// Main compilation routines
82//===========================================================================================================
83def build() {
84 debug = true
85 release = Settings.RunAllTests || Settings.RunBenchmark
86 Tools.BuildStage('Build : configure', true) {
87 // Configure must be run inside the tree
88 dir (SrcDir) {
89 // Generate the necessary build files
90 sh './autogen.sh'
91 }
92
93 // Build outside of the src tree to ease cleaning
94 dir (BuildDir) {
95 //Configure the compilation (Output is not relevant)
96 //Use the current directory as the installation target so nothing escapes the sandbox
97 //Also specify the compiler by hand
98 targets=""
99 if( Settings.RunAllTests || Settings.RunBenchmark ) {
100 targets="--with-target-hosts='host:debug,host:nodebug'"
101 } else {
102 targets="--with-target-hosts='host:debug'"
103 }
104
105 ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
106
107 sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
108
109 // Configure libcfa
110 sh 'make -j 8 --no-print-directory configure-libcfa'
111 }
112 }
113
114 Tools.BuildStage('Build : cfa-cpp', true) {
115 // Build outside of the src tree to ease cleaning
116 dir (BuildDir) {
117 // Build driver
118 sh 'make -j 8 --no-print-directory -C driver'
119
120 // Build translator
121 sh 'make -j 8 --no-print-directory -C src'
122 }
123 }
124
125 Tools.BuildStage('Build : libcfa(debug)', debug) {
126 // Build outside of the src tree to ease cleaning
127 dir (BuildDir) {
128 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
129 }
130 }
131
132 Tools.BuildStage('Build : libcfa(nodebug)', release) {
133 // Build outside of the src tree to ease cleaning
134 dir (BuildDir) {
135 sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
136 }
137 }
138
139 Tools.BuildStage('Build : install', true) {
140 // Build outside of the src tree to ease cleaning
141 dir (BuildDir) {
142 sh "make -j 8 --no-print-directory install"
143 }
144 }
145}
146
147def test() {
148 try {
149 Tools.BuildStage('Test: short', !Settings.RunAllTests) {
150 dir (BuildDir) {
151 //Run the tests from the tests directory
152 sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
153 }
154 }
155
156 Tools.BuildStage('Test: full', Settings.RunAllTests) {
157 dir (BuildDir) {
158 //Run the tests from the tests directory
159 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
160 sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
161 }
162 }
163 }
164 catch (Exception err) {
165 echo "Archiving core dumps"
166 dir (BuildDir) {
167 archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
168 }
169 throw err
170 }
171}
172
173def benchmark() {
174 Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
175 dir (BuildDir) {
176 //Append bench results
177 sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
178 }
179 }
180}
181
182def build_doc() {
183 Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
184 dir ('doc/user') {
185 make_doc()
186 }
187
188 dir ('doc/refrat') {
189 make_doc()
190 }
191 }
192}
193
194def publish() {
195 Tools.BuildStage('Publish', true) {
196
197 if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
198
199 def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
200 def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
201
202 //Then publish the results
203 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
204 do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
205 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
206 do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
207 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
208 do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
209 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
210 do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
211 }
212}
213
214//===========================================================================================================
215//Routine responsible of sending the email notification once the build is completed
216//===========================================================================================================
217//Standard build email notification
218def email(boolean log) {
219 //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
220 //Configurations for email format
221 echo 'Notifying users of result'
222
223 def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
224 def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
225 def email_body = """<p>This is an automated email from the Jenkins build machine. It was
226generated because of a git hooks/post-receive script following
227a ref change which was pushed to the C\u2200 repository.</p>
228""" + Tools.GitLogMessage()
229
230 def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
231
232 if( Settings && !Settings.Silent ) {
233 //send email notification
234 emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
235 } else {
236 echo "Would send email to: ${email_to}"
237 echo "With title: ${email_subject}"
238 echo "Content: \n${email_body}"
239 }
240}
241
242//===========================================================================================================
243// Helper classes/variables/routines
244//===========================================================================================================
245//Description of a compiler (Must be serializable since pipelines are persistent)
246class CC_Desc implements Serializable {
247 public String name
248 public String CXX
249 public String CC
250 public String lto
251
252 CC_Desc(String name, String CXX, String CC, String lto) {
253 this.name = name
254 this.CXX = CXX
255 this.CC = CC
256 this.lto = lto
257 }
258}
259
260//Description of an architecture (Must be serializable since pipelines are persistent)
261class Arch_Desc implements Serializable {
262 public String name
263 public String flags
264 public String node
265
266 Arch_Desc(String name, String flags, String node) {
267 this.name = name
268 this.flags = flags
269 this.node = node
270 }
271}
272
273class BuildSettings implements Serializable {
274 public final CC_Desc Compiler
275 public final Arch_Desc Architecture
276 public final Boolean NewAST
277 public final Boolean RunAllTests
278 public final Boolean RunBenchmark
279 public final Boolean BuildDocumentation
280 public final Boolean Publish
281 public final Boolean Silent
282 public final Boolean IsSandbox
283 public final String DescLong
284 public final String DescShort
285
286 public String GitNewRef
287 public String GitOldRef
288
289 BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
290 switch( param.Compiler ) {
291 case 'gcc-9':
292 this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
293 break
294 case 'gcc-8':
295 this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
296 break
297 case 'gcc-7':
298 this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
299 break
300 case 'gcc-6':
301 this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
302 break
303 case 'gcc-5':
304 this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
305 break
306 case 'gcc-4.9':
307 this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
308 break
309 case 'clang':
310 this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
311 break
312 default :
313 error "Unhandled compiler : ${cc}"
314 }
315
316 switch( param.Architecture ) {
317 case 'x64':
318 this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
319 break
320 case 'x86':
321 this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
322 break
323 default :
324 error "Unhandled architecture : ${arch}"
325 }
326
327 this.IsSandbox = (branch == "jenkins-sandbox")
328 this.NewAST = param.NewAST
329 this.RunAllTests = param.RunAllTests
330 this.RunBenchmark = param.RunBenchmark
331 this.BuildDocumentation = param.BuildDocumentation
332 this.Publish = param.Publish
333 this.Silent = param.Silent
334
335 def full = param.RunAllTests ? " (Full)" : ""
336 this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
337
338 final ast = this.NewAST ? "New AST" : "Old AST"
339 this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
340AST Version : ${ ast.toString() }
341Architecture : ${ this.Architecture.name }
342Arc Flags : ${ this.Architecture.flags }
343Run All Tests : ${ this.RunAllTests.toString() }
344Run Benchmark : ${ this.RunBenchmark.toString() }
345Build Documentation : ${ this.BuildDocumentation.toString() }
346Publish : ${ this.Publish.toString() }
347Silent : ${ this.Silent.toString() }
348"""
349
350 this.GitNewRef = ''
351 this.GitOldRef = ''
352 }
353}
354
355class PlotGroup implements Serializable {
356 public String name
357 public String unit
358 public boolean log
359
360 PlotGroup(String name, String unit, boolean log) {
361 this.name = name
362 this.unit = unit
363 this.log = log
364 }
365}
366
367def prepare_build() {
368 // prepare the properties
369 properties ([ \
370 buildDiscarder(logRotator( \
371 artifactDaysToKeepStr: '', \
372 artifactNumToKeepStr: '', \
373 daysToKeepStr: '730', \
374 numToKeepStr: '1000' \
375 )), \
376 [$class: 'ParametersDefinitionProperty', \
377 parameterDefinitions: [ \
378 [$class: 'ChoiceParameterDefinition', \
379 description: 'Which compiler to use', \
380 name: 'Compiler', \
381 choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
382 defaultValue: 'gcc-8', \
383 ], \
384 [$class: 'ChoiceParameterDefinition', \
385 description: 'The target architecture', \
386 name: 'Architecture', \
387 choices: 'x64\nx86', \
388 defaultValue: 'x64', \
389 ], \
390 [$class: 'BooleanParameterDefinition', \
391 description: 'If true, build compiler using new AST', \
392 name: 'NewAST', \
393 defaultValue: true, \
394 ], \
395 [$class: 'BooleanParameterDefinition', \
396 description: 'If false, only the quick test suite is ran', \
397 name: 'RunAllTests', \
398 defaultValue: false, \
399 ], \
400 [$class: 'BooleanParameterDefinition', \
401 description: 'If true, jenkins also runs benchmarks', \
402 name: 'RunBenchmark', \
403 defaultValue: false, \
404 ], \
405 [$class: 'BooleanParameterDefinition', \
406 description: 'If true, jenkins also builds documentation', \
407 name: 'BuildDocumentation', \
408 defaultValue: true, \
409 ], \
410 [$class: 'BooleanParameterDefinition', \
411 description: 'If true, jenkins also publishes results', \
412 name: 'Publish', \
413 defaultValue: false, \
414 ], \
415 [$class: 'BooleanParameterDefinition', \
416 description: 'If true, jenkins will not send emails', \
417 name: 'Silent', \
418 defaultValue: false, \
419 ], \
420 ],
421 ]])
422
423 // It's unfortunate but it looks like we need to checkout the entire repo just to get
424 // - the pretty git printer
425 // - Jenkins.tools
426 checkout scm
427
428 Tools = load "Jenkins/tools.groovy"
429
430 final settings = new BuildSettings(params, env.BRANCH_NAME)
431
432 currentBuild.description = settings.DescShort
433 echo settings.DescLong
434
435 return settings
436}
437
438def make_doc() {
439 def err = null
440 try {
441 sh 'make clean > /dev/null'
442 sh 'make > /dev/null 2>&1'
443 }
444 catch (Exception caughtError) {
445 err = caughtError //rethrow error later
446 sh 'cat build/*.log'
447 }
448 finally {
449 if (err) throw err // Must re-throw exception to propagate error
450 }
451}
452
453def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
454
455 if(new_data) {
456 echo "Publishing new data"
457 }
458
459 def series = new_data ? [[
460 file: "${file}.csv",
461 exclusionValues: '',
462 displayTableFlag: false,
463 inclusionFlag: 'OFF',
464 url: ''
465 ]] : [];
466
467 echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
468 dir("${BuildDir}/benchmark/") {
469 plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
470 csvSeries: series,
471 group: "${group.name}",
472 title: "${title}",
473 style: 'lineSimple',
474 exclZero: false,
475 keepRecords: false,
476 logarithmic: !relative && group.log,
477 numBuilds: '120',
478 useDescr: true,
479 yaxis: group.unit,
480 yaxisMaximum: '',
481 yaxisMinimum: ''
482 }
483}
Note: See TracBrowser for help on using the repository browser.