1 | #!groovy
|
---|
2 |
|
---|
3 | import groovy.transform.Field
|
---|
4 |
|
---|
5 | //===========================================================================================================
|
---|
6 | // Main loop of the compilation
|
---|
7 | //===========================================================================================================
|
---|
8 |
|
---|
9 | node('master') {
|
---|
10 | // Globals
|
---|
11 | BuildDir = pwd tmp: true
|
---|
12 | SrcDir = pwd tmp: false
|
---|
13 | Settings = null
|
---|
14 | Tools = null
|
---|
15 |
|
---|
16 | // Local variables
|
---|
17 | def err = null
|
---|
18 | def log_needed = false
|
---|
19 |
|
---|
20 | currentBuild.result = "SUCCESS"
|
---|
21 |
|
---|
22 | try {
|
---|
23 | //Wrap build to add timestamp to command line
|
---|
24 | wrap([$class: 'TimestamperBuildWrapper']) {
|
---|
25 |
|
---|
26 | Settings = prepare_build()
|
---|
27 |
|
---|
28 | node(Settings.Architecture.node) {
|
---|
29 | BuildDir = pwd tmp: true
|
---|
30 | SrcDir = pwd tmp: false
|
---|
31 |
|
---|
32 | Tools.Clean()
|
---|
33 |
|
---|
34 | Tools.Checkout()
|
---|
35 |
|
---|
36 | build()
|
---|
37 |
|
---|
38 | test()
|
---|
39 |
|
---|
40 | benchmark()
|
---|
41 |
|
---|
42 | build_doc()
|
---|
43 |
|
---|
44 | publish()
|
---|
45 | }
|
---|
46 |
|
---|
47 | // Update the build directories when exiting the node
|
---|
48 | BuildDir = pwd tmp: true
|
---|
49 | SrcDir = pwd tmp: false
|
---|
50 | }
|
---|
51 | }
|
---|
52 |
|
---|
53 | //If an exception is caught we need to change the status and remember to
|
---|
54 | //attach the build log to the email
|
---|
55 | catch (Exception caughtError) {
|
---|
56 | // Store the result of the build log
|
---|
57 | currentBuild.result = "FAILURE"
|
---|
58 |
|
---|
59 | // An error has occured, the build log is relevent
|
---|
60 | log_needed = true
|
---|
61 |
|
---|
62 | // rethrow error later
|
---|
63 | err = caughtError
|
---|
64 |
|
---|
65 | // print the error so it shows in the log
|
---|
66 | echo err.toString()
|
---|
67 | }
|
---|
68 |
|
---|
69 | finally {
|
---|
70 | //Send email with final results if this is not a full build
|
---|
71 | email(log_needed)
|
---|
72 |
|
---|
73 | echo 'Build Completed'
|
---|
74 |
|
---|
75 | /* Must re-throw exception to propagate error */
|
---|
76 | if (err) {
|
---|
77 | throw err
|
---|
78 | }
|
---|
79 | }
|
---|
80 | }
|
---|
81 | //===========================================================================================================
|
---|
82 | // Main compilation routines
|
---|
83 | //===========================================================================================================
|
---|
84 | def build() {
|
---|
85 | debug = true
|
---|
86 | release = Settings.RunAllTests || Settings.RunBenchmark
|
---|
87 | Tools.BuildStage('Build : configure', true) {
|
---|
88 | // Configure must be run inside the tree
|
---|
89 | dir (SrcDir) {
|
---|
90 | // Generate the necessary build files
|
---|
91 | sh './autogen.sh'
|
---|
92 | }
|
---|
93 |
|
---|
94 | // Build outside of the src tree to ease cleaning
|
---|
95 | dir (BuildDir) {
|
---|
96 | //Configure the compilation (Output is not relevant)
|
---|
97 | //Use the current directory as the installation target so nothing escapes the sandbox
|
---|
98 | //Also specify the compiler by hand
|
---|
99 | targets=""
|
---|
100 | if( Settings.RunAllTests || Settings.RunBenchmark ) {
|
---|
101 | targets="--with-target-hosts='host:debug,host:nodebug'"
|
---|
102 | } else {
|
---|
103 | targets="--with-target-hosts='host:debug'"
|
---|
104 | }
|
---|
105 |
|
---|
106 | ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
|
---|
107 |
|
---|
108 | sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
|
---|
109 |
|
---|
110 | // Configure libcfa
|
---|
111 | sh 'make -j 8 --no-print-directory configure-libcfa'
|
---|
112 | }
|
---|
113 | }
|
---|
114 |
|
---|
115 | Tools.BuildStage('Build : cfa-cpp', true) {
|
---|
116 | // Build outside of the src tree to ease cleaning
|
---|
117 | dir (BuildDir) {
|
---|
118 | // Build driver
|
---|
119 | sh 'make -j 8 --no-print-directory -C driver'
|
---|
120 |
|
---|
121 | // Build translator
|
---|
122 | sh 'make -j 8 --no-print-directory -C src'
|
---|
123 | }
|
---|
124 | }
|
---|
125 |
|
---|
126 | Tools.BuildStage('Build : libcfa(debug)', debug) {
|
---|
127 | // Build outside of the src tree to ease cleaning
|
---|
128 | dir (BuildDir) {
|
---|
129 | sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
|
---|
130 | }
|
---|
131 | }
|
---|
132 |
|
---|
133 | Tools.BuildStage('Build : libcfa(nodebug)', release) {
|
---|
134 | // Build outside of the src tree to ease cleaning
|
---|
135 | dir (BuildDir) {
|
---|
136 | sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
|
---|
137 | }
|
---|
138 | }
|
---|
139 |
|
---|
140 | Tools.BuildStage('Build : install', true) {
|
---|
141 | // Build outside of the src tree to ease cleaning
|
---|
142 | dir (BuildDir) {
|
---|
143 | sh "make -j 8 --no-print-directory install"
|
---|
144 | }
|
---|
145 | }
|
---|
146 | }
|
---|
147 |
|
---|
148 | def test() {
|
---|
149 | try {
|
---|
150 | Tools.BuildStage('Test: short', !Settings.RunAllTests) {
|
---|
151 | dir (BuildDir) {
|
---|
152 | //Run the tests from the tests directory
|
---|
153 | sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
|
---|
154 | }
|
---|
155 | }
|
---|
156 |
|
---|
157 | Tools.BuildStage('Test: full', Settings.RunAllTests) {
|
---|
158 | dir (BuildDir) {
|
---|
159 | //Run the tests from the tests directory
|
---|
160 | sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
|
---|
161 | sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
|
---|
162 | }
|
---|
163 | }
|
---|
164 | }
|
---|
165 | catch (Exception err) {
|
---|
166 | echo "Archiving core dumps"
|
---|
167 | dir (BuildDir) {
|
---|
168 | archiveArtifacts artifacts: "tests/crashes/**/*,lib/**/lib*.so*", fingerprint: true
|
---|
169 | }
|
---|
170 | throw err
|
---|
171 | }
|
---|
172 | }
|
---|
173 |
|
---|
174 | def benchmark() {
|
---|
175 | Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
|
---|
176 | dir (BuildDir) {
|
---|
177 | //Append bench results
|
---|
178 | sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
|
---|
179 | }
|
---|
180 | }
|
---|
181 | }
|
---|
182 |
|
---|
183 | def build_doc() {
|
---|
184 | Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
|
---|
185 | dir ('doc/user') {
|
---|
186 | make_doc()
|
---|
187 | }
|
---|
188 |
|
---|
189 | dir ('doc/refrat') {
|
---|
190 | make_doc()
|
---|
191 | }
|
---|
192 | }
|
---|
193 | }
|
---|
194 |
|
---|
195 | def publish() {
|
---|
196 | Tools.BuildStage('Publish', true) {
|
---|
197 |
|
---|
198 | if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
|
---|
199 |
|
---|
200 | def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
|
---|
201 | def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
|
---|
202 |
|
---|
203 | //Then publish the results
|
---|
204 | do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
|
---|
205 | do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
|
---|
206 | do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
|
---|
207 | do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
|
---|
208 | do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
|
---|
209 | do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
|
---|
210 | do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
|
---|
211 | do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
|
---|
212 | }
|
---|
213 | }
|
---|
214 |
|
---|
215 | //===========================================================================================================
|
---|
216 | //Routine responsible of sending the email notification once the build is completed
|
---|
217 | //===========================================================================================================
|
---|
218 | //Standard build email notification
|
---|
219 | def email(boolean log) {
|
---|
220 | //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
|
---|
221 | //Configurations for email format
|
---|
222 | echo 'Notifying users of result'
|
---|
223 |
|
---|
224 | def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
|
---|
225 | def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
|
---|
226 | def email_body = """<p>This is an automated email from the Jenkins build machine. It was
|
---|
227 | generated because of a git hooks/post-receive script following
|
---|
228 | a ref change which was pushed to the C\u2200 repository.</p>
|
---|
229 | """ + Tools.GitLogMessage()
|
---|
230 |
|
---|
231 | def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
|
---|
232 |
|
---|
233 | if( Settings && !Settings.Silent ) {
|
---|
234 | //send email notification
|
---|
235 | emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
|
---|
236 | } else {
|
---|
237 | echo "Would send email to: ${email_to}"
|
---|
238 | echo "With title: ${email_subject}"
|
---|
239 | echo "Content: \n${email_body}"
|
---|
240 | }
|
---|
241 | }
|
---|
242 |
|
---|
243 | //===========================================================================================================
|
---|
244 | // Helper classes/variables/routines
|
---|
245 | //===========================================================================================================
|
---|
246 | //Description of a compiler (Must be serializable since pipelines are persistent)
|
---|
247 | class CC_Desc implements Serializable {
|
---|
248 | public String name
|
---|
249 | public String CXX
|
---|
250 | public String CC
|
---|
251 | public String lto
|
---|
252 |
|
---|
253 | CC_Desc(String name, String CXX, String CC, String lto) {
|
---|
254 | this.name = name
|
---|
255 | this.CXX = CXX
|
---|
256 | this.CC = CC
|
---|
257 | this.lto = lto
|
---|
258 | }
|
---|
259 | }
|
---|
260 |
|
---|
261 | //Description of an architecture (Must be serializable since pipelines are persistent)
|
---|
262 | class Arch_Desc implements Serializable {
|
---|
263 | public String name
|
---|
264 | public String flags
|
---|
265 | public String node
|
---|
266 |
|
---|
267 | Arch_Desc(String name, String flags, String node) {
|
---|
268 | this.name = name
|
---|
269 | this.flags = flags
|
---|
270 | this.node = node
|
---|
271 | }
|
---|
272 | }
|
---|
273 |
|
---|
274 | class BuildSettings implements Serializable {
|
---|
275 | public final CC_Desc Compiler
|
---|
276 | public final Arch_Desc Architecture
|
---|
277 | public final Boolean NewAST
|
---|
278 | public final Boolean RunAllTests
|
---|
279 | public final Boolean RunBenchmark
|
---|
280 | public final Boolean BuildDocumentation
|
---|
281 | public final Boolean Publish
|
---|
282 | public final Boolean Silent
|
---|
283 | public final Boolean IsSandbox
|
---|
284 | public final String DescLong
|
---|
285 | public final String DescShort
|
---|
286 |
|
---|
287 | public String GitNewRef
|
---|
288 | public String GitOldRef
|
---|
289 |
|
---|
290 | BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
|
---|
291 | switch( param.Compiler ) {
|
---|
292 | case 'gcc-9':
|
---|
293 | this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
|
---|
294 | break
|
---|
295 | case 'gcc-8':
|
---|
296 | this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
|
---|
297 | break
|
---|
298 | case 'gcc-7':
|
---|
299 | this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
|
---|
300 | break
|
---|
301 | case 'gcc-6':
|
---|
302 | this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
|
---|
303 | break
|
---|
304 | case 'gcc-5':
|
---|
305 | this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
|
---|
306 | break
|
---|
307 | case 'gcc-4.9':
|
---|
308 | this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
|
---|
309 | break
|
---|
310 | case 'clang':
|
---|
311 | this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
|
---|
312 | break
|
---|
313 | default :
|
---|
314 | error "Unhandled compiler : ${cc}"
|
---|
315 | }
|
---|
316 |
|
---|
317 | switch( param.Architecture ) {
|
---|
318 | case 'x64':
|
---|
319 | this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
|
---|
320 | break
|
---|
321 | case 'x86':
|
---|
322 | this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
|
---|
323 | break
|
---|
324 | default :
|
---|
325 | error "Unhandled architecture : ${arch}"
|
---|
326 | }
|
---|
327 |
|
---|
328 | this.IsSandbox = (branch == "jenkins-sandbox")
|
---|
329 | this.NewAST = param.NewAST
|
---|
330 | this.RunAllTests = param.RunAllTests
|
---|
331 | this.RunBenchmark = param.RunBenchmark
|
---|
332 | this.BuildDocumentation = param.BuildDocumentation
|
---|
333 | this.Publish = param.Publish
|
---|
334 | this.Silent = param.Silent
|
---|
335 |
|
---|
336 | def full = param.RunAllTests ? " (Full)" : ""
|
---|
337 | this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
|
---|
338 |
|
---|
339 | final ast = this.NewAST ? "New AST" : "Old AST"
|
---|
340 | this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
|
---|
341 | AST Version : ${ ast.toString() }
|
---|
342 | Architecture : ${ this.Architecture.name }
|
---|
343 | Arc Flags : ${ this.Architecture.flags }
|
---|
344 | Run All Tests : ${ this.RunAllTests.toString() }
|
---|
345 | Run Benchmark : ${ this.RunBenchmark.toString() }
|
---|
346 | Build Documentation : ${ this.BuildDocumentation.toString() }
|
---|
347 | Publish : ${ this.Publish.toString() }
|
---|
348 | Silent : ${ this.Silent.toString() }
|
---|
349 | """
|
---|
350 |
|
---|
351 | this.GitNewRef = ''
|
---|
352 | this.GitOldRef = ''
|
---|
353 | }
|
---|
354 | }
|
---|
355 |
|
---|
356 | class PlotGroup implements Serializable {
|
---|
357 | public String name
|
---|
358 | public String unit
|
---|
359 | public boolean log
|
---|
360 |
|
---|
361 | PlotGroup(String name, String unit, boolean log) {
|
---|
362 | this.name = name
|
---|
363 | this.unit = unit
|
---|
364 | this.log = log
|
---|
365 | }
|
---|
366 | }
|
---|
367 |
|
---|
368 | def prepare_build() {
|
---|
369 | // prepare the properties
|
---|
370 | properties ([ \
|
---|
371 | buildDiscarder(logRotator( \
|
---|
372 | artifactDaysToKeepStr: '', \
|
---|
373 | artifactNumToKeepStr: '', \
|
---|
374 | daysToKeepStr: '730', \
|
---|
375 | numToKeepStr: '1000' \
|
---|
376 | )), \
|
---|
377 | [$class: 'ParametersDefinitionProperty', \
|
---|
378 | parameterDefinitions: [ \
|
---|
379 | [$class: 'ChoiceParameterDefinition', \
|
---|
380 | description: 'Which compiler to use', \
|
---|
381 | name: 'Compiler', \
|
---|
382 | choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
|
---|
383 | defaultValue: 'gcc-8', \
|
---|
384 | ], \
|
---|
385 | [$class: 'ChoiceParameterDefinition', \
|
---|
386 | description: 'The target architecture', \
|
---|
387 | name: 'Architecture', \
|
---|
388 | choices: 'x64\nx86', \
|
---|
389 | defaultValue: 'x64', \
|
---|
390 | ], \
|
---|
391 | [$class: 'BooleanParameterDefinition', \
|
---|
392 | description: 'If true, build compiler using new AST', \
|
---|
393 | name: 'NewAST', \
|
---|
394 | defaultValue: true, \
|
---|
395 | ], \
|
---|
396 | [$class: 'BooleanParameterDefinition', \
|
---|
397 | description: 'If false, only the quick test suite is ran', \
|
---|
398 | name: 'RunAllTests', \
|
---|
399 | defaultValue: false, \
|
---|
400 | ], \
|
---|
401 | [$class: 'BooleanParameterDefinition', \
|
---|
402 | description: 'If true, jenkins also runs benchmarks', \
|
---|
403 | name: 'RunBenchmark', \
|
---|
404 | defaultValue: false, \
|
---|
405 | ], \
|
---|
406 | [$class: 'BooleanParameterDefinition', \
|
---|
407 | description: 'If true, jenkins also builds documentation', \
|
---|
408 | name: 'BuildDocumentation', \
|
---|
409 | defaultValue: true, \
|
---|
410 | ], \
|
---|
411 | [$class: 'BooleanParameterDefinition', \
|
---|
412 | description: 'If true, jenkins also publishes results', \
|
---|
413 | name: 'Publish', \
|
---|
414 | defaultValue: false, \
|
---|
415 | ], \
|
---|
416 | [$class: 'BooleanParameterDefinition', \
|
---|
417 | description: 'If true, jenkins will not send emails', \
|
---|
418 | name: 'Silent', \
|
---|
419 | defaultValue: false, \
|
---|
420 | ], \
|
---|
421 | ],
|
---|
422 | ]])
|
---|
423 |
|
---|
424 | // It's unfortunate but it looks like we need to checkout the entire repo just to get
|
---|
425 | // - the pretty git printer
|
---|
426 | // - Jenkins.tools
|
---|
427 | checkout scm
|
---|
428 |
|
---|
429 | Tools = load "Jenkins/tools.groovy"
|
---|
430 |
|
---|
431 | final settings = new BuildSettings(params, env.BRANCH_NAME)
|
---|
432 |
|
---|
433 | currentBuild.description = settings.DescShort
|
---|
434 | echo settings.DescLong
|
---|
435 |
|
---|
436 | return settings
|
---|
437 | }
|
---|
438 |
|
---|
439 | def make_doc() {
|
---|
440 | def err = null
|
---|
441 | try {
|
---|
442 | sh 'make clean > /dev/null'
|
---|
443 | sh 'make > /dev/null 2>&1'
|
---|
444 | }
|
---|
445 | catch (Exception caughtError) {
|
---|
446 | err = caughtError //rethrow error later
|
---|
447 | sh 'cat build/*.log'
|
---|
448 | }
|
---|
449 | finally {
|
---|
450 | if (err) throw err // Must re-throw exception to propagate error
|
---|
451 | }
|
---|
452 | }
|
---|
453 |
|
---|
454 | def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
|
---|
455 |
|
---|
456 | if(new_data) {
|
---|
457 | echo "Publishing new data"
|
---|
458 | }
|
---|
459 |
|
---|
460 | def series = new_data ? [[
|
---|
461 | file: "${file}.csv",
|
---|
462 | exclusionValues: '',
|
---|
463 | displayTableFlag: false,
|
---|
464 | inclusionFlag: 'OFF',
|
---|
465 | url: ''
|
---|
466 | ]] : [];
|
---|
467 |
|
---|
468 | echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
|
---|
469 | dir("${BuildDir}/benchmark/") {
|
---|
470 | plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
|
---|
471 | csvSeries: series,
|
---|
472 | group: "${group.name}",
|
---|
473 | title: "${title}",
|
---|
474 | style: 'lineSimple',
|
---|
475 | exclZero: false,
|
---|
476 | keepRecords: false,
|
---|
477 | logarithmic: !relative && group.log,
|
---|
478 | numBuilds: '120',
|
---|
479 | useDescr: true,
|
---|
480 | yaxis: group.unit,
|
---|
481 | yaxisMaximum: '',
|
---|
482 | yaxisMinimum: ''
|
---|
483 | }
|
---|
484 | }
|
---|