Context Navigation

source: Jenkinsfile @ 9c5aef9

ADTast-experimentalenumpthread-emulationqualifiedEnum

Last change on this file since 9c5aef9 was 0240a7cb, checked in by Thierry Delisle <tdelisle@…>, 3 years ago
Apparently -j1 is a special case I don't support, changed it to -j2.
Property mode set to `100644`
File size: 15.6 KB

Rev	Line
[a63ad80]	1	#!groovy
	2
[7a230fd]	3	import groovy.transform.Field
	4
[29f4fe62]	5	//===========================================================================================================
[9beae23]	6	// Main loop of the compilation
[29f4fe62]	7	//===========================================================================================================
[56a9ce6]	8
[d3b95f1]	9	// Globals
[e018546]	10	BuildDir = null
	11	SrcDir = null
[d3b95f1]	12	Settings = null
	13	Tools = null
[4f9e706]	14
[d3b95f1]	15	// Local variables
	16	def err = null
	17	def log_needed = false
[5b8413b4]	18
[d3b95f1]	19	currentBuild.result = "SUCCESS"
[4f9e706]	20
[d3b95f1]	21	try {
[47138ee]	22	node {
	23	//Wrap build to add timestamp to command line
	24	wrap([$class: 'TimestamperBuildWrapper']) {
[85142648]	25	Settings = prepare_build()
[d3b95f1]	26	}
[47138ee]	27	}
[952ee7a]	28
[47138ee]	29	node(Settings.Architecture.node) {
	30	//Wrap build to add timestamp to command line
	31	wrap([$class: 'TimestamperBuildWrapper']) {
[d3b95f1]	32	BuildDir = pwd tmp: true
	33	SrcDir = pwd tmp: false
	34	currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
[7aebc62]	35
[d3b95f1]	36	Tools.Clean()
[fde808df]	37
[d3b95f1]	38	Tools.Checkout()
[95fdb0a]	39
[d3b95f1]	40	build()
[95fdb0a]	41
[d3b95f1]	42	test()
[95fdb0a]	43
[d3b95f1]	44	benchmark()
[7359098]	45
[d3b95f1]	46	build_doc()
[9beae23]	47
[d3b95f1]	48	publish()
[9beae23]	49	}
[738cf8f]	50	}
[d3b95f1]	51	}
[738cf8f]	52
[d3b95f1]	53	//If an exception is caught we need to change the status and remember to
	54	//attach the build log to the email
	55	catch (Exception caughtError) {
	56	// Store the result of the build log
	57	currentBuild.result = "FAILURE"
[9dd31e7]	58
[d3b95f1]	59	// An error has occured, the build log is relevent
	60	log_needed = true
[9beae23]	61
[d3b95f1]	62	// rethrow error later
	63	err = caughtError
[852ae0ea]	64
[d3b95f1]	65	// print the error so it shows in the log
	66	echo err.toString()
	67	}
[738cf8f]	68
[d3b95f1]	69	finally {
	70	//Send email with final results if this is not a full build
	71	email(log_needed)
[9beae23]	72
[d3b95f1]	73	echo 'Build Completed'
[734891d]	74
[d3b95f1]	75	/* Must re-throw exception to propagate error */
	76	if (err) {
	77	throw err
[738cf8f]	78	}
	79	}
[29f4fe62]	80	//===========================================================================================================
[9beae23]	81	// Main compilation routines
[29f4fe62]	82	//===========================================================================================================
[95fdb0a]	83	def build() {
[e507c11]	84	debug = true
	85	release = Settings.RunAllTests \|\| Settings.RunBenchmark
[91aa5ab]	86	Tools.BuildStage('Build : configure', true) {
[dcf1979]	87	// Configure must be run inside the tree
	88	dir (SrcDir) {
	89	// Generate the necessary build files
	90	sh './autogen.sh'
	91	}
	92
[50f2cfc]	93	// Build outside of the src tree to ease cleaning
[5b8413b4]	94	dir (BuildDir) {
[8e58264]	95	//Configure the compilation (Output is not relevant)
[50f2cfc]	96	//Use the current directory as the installation target so nothing escapes the sandbox
	97	//Also specify the compiler by hand
[3fc5f010]	98	targets=""
[d4510ea]	99	if( Settings.RunAllTests \|\| Settings.RunBenchmark ) {
[6bde81d]	100	targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]	101	} else {
[6bde81d]	102	targets="--with-target-hosts='host:debug'"
[3fc5f010]	103	}
	104
[d21dd3cb]	105	ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
	106
	107	sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]	108
	109	// Configure libcfa
[e70e54e]	110	sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]	111	}
	112	}
	113
[91aa5ab]	114	Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]	115	// Build outside of the src tree to ease cleaning
	116	dir (BuildDir) {
	117	// Build driver
	118	sh 'make -j 8 --no-print-directory -C driver'
	119
	120	// Build translator
	121	sh 'make -j 8 --no-print-directory -C src'
	122	}
	123	}
[1752d0e]	124
[91aa5ab]	125	Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]	126	// Build outside of the src tree to ease cleaning
	127	dir (BuildDir) {
	128	sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
	129	}
	130	}
	131
[91aa5ab]	132	Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]	133	// Build outside of the src tree to ease cleaning
	134	dir (BuildDir) {
	135	sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]	136	}
[620dd2b]	137	}
[aa96fba]	138
[91aa5ab]	139	Tools.BuildStage('Build : install', true) {
[aa96fba]	140	// Build outside of the src tree to ease cleaning
	141	dir (BuildDir) {
	142	sh "make -j 8 --no-print-directory install"
	143	}
	144	}
[95fdb0a]	145	}
[24eecab]	146
[95fdb0a]	147	def test() {
[4c1b9ea8]	148	try {
[1baf6ed]	149	// Print potential limits before testing
	150	// in case jenkins messes with them
[a33dcd5]	151	sh 'free -h'
[1baf6ed]	152	sh 'ulimit -a'
	153
[91aa5ab]	154	Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]	155	dir (BuildDir) {
[3e93c00]	156	//Run the tests from the tests directory
[4c51aca]	157	sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]	158	}
[4c1b9ea8]	159	}
	160
[91aa5ab]	161	Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]	162	dir (BuildDir) {
[7d84369]	163	jopt = ""
	164	if( Settings.Architecture.node == 'x86' ) {
[0240a7cb]	165	jopt = "-j2"
[7d84369]	166	}
[4c1b9ea8]	167	//Run the tests from the tests directory
[7d84369]	168	sh """make ${jopt} --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
	169	sh """make ${jopt} --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]	170	}
[9beae23]	171	}
[620dd2b]	172	}
[4c1b9ea8]	173	catch (Exception err) {
	174	echo "Archiving core dumps"
[c95fdc9]	175	dir (BuildDir) {
[cc9ec56]	176	def exists = fileExists 'tests/crashes'
	177	if( exists ) {
[cece53c]	178	sh """${SrcDir}/tools/jenkins/archive-gen.sh"""
	179	archiveArtifacts artifacts: "tests/crashes/*/,lib/*/lib.so*,setup.sh", fingerprint: true
	180	}
[c95fdc9]	181	}
[4c1b9ea8]	182	throw err
	183	}
[95fdb0a]	184	}
	185
	186	def benchmark() {
[91aa5ab]	187	Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]	188	dir (BuildDir) {
[0dc3ac3]	189	//Append bench results
[c6f1f3e]	190	sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]	191	}
[620dd2b]	192	}
[9beae23]	193	}
[efd60d67]	194
[95fdb0a]	195	def build_doc() {
[91aa5ab]	196	Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]	197	dir ('doc/user') {
	198	make_doc()
	199	}
	200
	201	dir ('doc/refrat') {
	202	make_doc()
	203	}
[620dd2b]	204	}
[9beae23]	205	}
	206
[95fdb0a]	207	def publish() {
[91aa5ab]	208	Tools.BuildStage('Publish', true) {
[95fdb0a]	209
[1b3eef8]	210	if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]	211
[3221a2b]	212	def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
	213	def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]	214
[3898392]	215	//Then publish the results
[13d2dac]	216	do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
	217	do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
	218	do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
	219	do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
	220	do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
	221	do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
	222	do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
	223	do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]	224	}
[95fdb0a]	225	}
	226
[29f4fe62]	227	//===========================================================================================================
	228	//Routine responsible of sending the email notification once the build is completed
	229	//===========================================================================================================
[a336d46]	230	//Standard build email notification
[13c98a4]	231	def email(boolean log) {
[bd50205]	232	node {
	233	//Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
	234	//Configurations for email format
	235	echo 'Notifying users of result'
	236
	237	def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
	238	def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
	239	def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[1781e97]	240	generated because of a git hooks/post-receive script following
	241	a ref change which was pushed to the C\u2200 repository.</p>
[9824500]	242
[1781e97]	243	<p>- Status --------------------------------------------------------------</p>
[9824500]	244
[1781e97]	245	<p>BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}</p>
	246	<p>Check console output at ${env.BUILD_URL} to view the results.</p>
	247	""" + Tools.GitLogMessage()
[bd50205]	248
	249	def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
	250
	251	if( Settings && !Settings.Silent ) {
	252	//send email notification
	253	emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
	254	} else {
	255	echo "Would send email to: ${email_to}"
	256	echo "With title: ${email_subject}"
	257	echo "Content: \n${email_body}"
	258	}
[094a42c]	259	}
[e8a22a7]	260	}
[5b8413b4]	261
	262	//===========================================================================================================
	263	// Helper classes/variables/routines
	264	//===========================================================================================================
	265	//Description of a compiler (Must be serializable since pipelines are persistent)
	266	class CC_Desc implements Serializable {
[93fe3154]	267	public String name
	268	public String CXX
[6ebc13f]	269	public String CC
[bf22bc6]	270	public String lto
[93fe3154]	271
[bf22bc6]	272	CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]	273	this.name = name
	274	this.CXX = CXX
[bf22bc6]	275	this.CC = CC
	276	this.lto = lto
[5b8413b4]	277	}
	278	}
	279
	280	//Description of an architecture (Must be serializable since pipelines are persistent)
	281	class Arch_Desc implements Serializable {
	282	public String name
	283	public String flags
[5307c33]	284	public String node
[5b8413b4]	285
[5307c33]	286	Arch_Desc(String name, String flags, String node) {
[5b8413b4]	287	this.name = name
	288	this.flags = flags
[5307c33]	289	this.node = node
[5b8413b4]	290	}
	291	}
	292
	293	class BuildSettings implements Serializable {
	294	public final CC_Desc Compiler
	295	public final Arch_Desc Architecture
[d21dd3cb]	296	public final Boolean NewAST
[5b8413b4]	297	public final Boolean RunAllTests
	298	public final Boolean RunBenchmark
	299	public final Boolean BuildDocumentation
	300	public final Boolean Publish
	301	public final Boolean Silent
	302	public final Boolean IsSandbox
	303	public final String DescLong
	304	public final String DescShort
	305
[a336d46]	306	public String GitNewRef
	307	public String GitOldRef
	308
[490cb3c]	309	BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]	310	switch( param.Compiler ) {
[8110bc3]	311	case 'gcc-11':
	312	this.Compiler = new CC_Desc('gcc-11', 'g++-11', 'gcc-11', '-flto=auto')
	313	break
	314	case 'gcc-10':
	315	this.Compiler = new CC_Desc('gcc-10', 'g++-10', 'gcc-10', '-flto=auto')
	316	break
[099f5bd]	317	case 'gcc-9':
[bf22bc6]	318	this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]	319	break
	320	case 'gcc-8':
[bf22bc6]	321	this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]	322	break
	323	case 'gcc-7':
[bf22bc6]	324	this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]	325	break
[5b8413b4]	326	case 'gcc-6':
[bf22bc6]	327	this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]	328	break
	329	case 'gcc-5':
[bf22bc6]	330	this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]	331	break
	332	case 'gcc-4.9':
[bf22bc6]	333	this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]	334	break
	335	case 'clang':
[8110bc3]	336	this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-10', '-flto=thin -flto-jobs=0')
[5b8413b4]	337	break
	338	default :
	339	error "Unhandled compiler : ${cc}"
	340	}
	341
	342	switch( param.Architecture ) {
	343	case 'x64':
[a3e8281]	344	this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]	345	break
	346	case 'x86':
[a3e8281]	347	this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]	348	break
	349	default :
	350	error "Unhandled architecture : ${arch}"
	351	}
	352
[f95e8f0]	353	this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]	354	this.NewAST = param.NewAST
[5b8413b4]	355	this.RunAllTests = param.RunAllTests
[7a230fd]	356	this.RunBenchmark = param.RunBenchmark
[5b8413b4]	357	this.BuildDocumentation = param.BuildDocumentation
[7a230fd]	358	this.Publish = param.Publish
[5b8413b4]	359	this.Silent = param.Silent
	360
	361	def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]	362	this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]	363
[fe3d9ab]	364	final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]	365	this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[85799aa]	366	AST Version : ${ ast.toString() }
[5b8413b4]	367	Architecture : ${ this.Architecture.name }
	368	Arc Flags : ${ this.Architecture.flags }
	369	Run All Tests : ${ this.RunAllTests.toString() }
	370	Run Benchmark : ${ this.RunBenchmark.toString() }
	371	Build Documentation : ${ this.BuildDocumentation.toString() }
	372	Publish : ${ this.Publish.toString() }
	373	Silent : ${ this.Silent.toString() }
	374	"""
[a336d46]	375
	376	this.GitNewRef = ''
	377	this.GitOldRef = ''
[5b8413b4]	378	}
	379	}
	380
[490cb3c]	381	class PlotGroup implements Serializable {
	382	public String name
	383	public String unit
	384	public boolean log
	385
	386	PlotGroup(String name, String unit, boolean log) {
	387	this.name = name
	388	this.unit = unit
	389	this.log = log
	390	}
	391	}
	392
[5b8413b4]	393	def prepare_build() {
	394	// prepare the properties
	395	properties ([ \
[62f96ae]	396	buildDiscarder(logRotator( \
	397	artifactDaysToKeepStr: '', \
	398	artifactNumToKeepStr: '', \
	399	daysToKeepStr: '730', \
	400	numToKeepStr: '1000' \
	401	)), \
[5b8413b4]	402	[$class: 'ParametersDefinitionProperty', \
	403	parameterDefinitions: [ \
	404	[$class: 'ChoiceParameterDefinition', \
	405	description: 'Which compiler to use', \
	406	name: 'Compiler', \
[b6f39aa]	407	choices: 'gcc-11\ngcc-10\ngcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]	408	defaultValue: 'gcc-8', \
[5b8413b4]	409	], \
	410	[$class: 'ChoiceParameterDefinition', \
	411	description: 'The target architecture', \
	412	name: 'Architecture', \
	413	choices: 'x64\nx86', \
	414	defaultValue: 'x64', \
	415	], \
[d21dd3cb]	416	[$class: 'BooleanParameterDefinition', \
	417	description: 'If true, build compiler using new AST', \
	418	name: 'NewAST', \
[6a531ab]	419	defaultValue: true, \
[d21dd3cb]	420	], \
[5b8413b4]	421	[$class: 'BooleanParameterDefinition', \
	422	description: 'If false, only the quick test suite is ran', \
	423	name: 'RunAllTests', \
	424	defaultValue: false, \
[fe3d9ab]	425	], \
[5b8413b4]	426	[$class: 'BooleanParameterDefinition', \
	427	description: 'If true, jenkins also runs benchmarks', \
	428	name: 'RunBenchmark', \
	429	defaultValue: false, \
	430	], \
	431	[$class: 'BooleanParameterDefinition', \
	432	description: 'If true, jenkins also builds documentation', \
	433	name: 'BuildDocumentation', \
	434	defaultValue: true, \
	435	], \
	436	[$class: 'BooleanParameterDefinition', \
	437	description: 'If true, jenkins also publishes results', \
	438	name: 'Publish', \
	439	defaultValue: false, \
	440	], \
	441	[$class: 'BooleanParameterDefinition', \
	442	description: 'If true, jenkins will not send emails', \
	443	name: 'Silent', \
	444	defaultValue: false, \
	445	], \
	446	],
	447	]])
	448
[bd8dca2]	449	// It's unfortunate but it looks like we need to checkout the entire repo just to get
	450	// - the pretty git printer
	451	// - Jenkins.tools
[4c55047]	452	checkout scm
[2407853]	453
[1483a16]	454	Tools = load "Jenkins/tools.groovy"
[bd8dca2]	455
[490cb3c]	456	final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]	457
	458	currentBuild.description = settings.DescShort
	459	echo settings.DescLong
	460
	461	return settings
	462	}
	463
	464	def make_doc() {
	465	def err = null
	466	try {
	467	sh 'make clean > /dev/null'
	468	sh 'make > /dev/null 2>&1'
	469	}
	470	catch (Exception caughtError) {
	471	err = caughtError //rethrow error later
[65f4a51]	472	sh 'cat build/*.log'
[5b8413b4]	473	}
	474	finally {
	475	if (err) throw err // Must re-throw exception to propagate error
	476	}
[a2a0065]	477	}
	478
[3221a2b]	479	def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]	480
[1b3eef8]	481	if(new_data) {
	482	echo "Publishing new data"
	483	}
	484
[cdcd53dc]	485	def series = new_data ? [[
[df57a84]	486	file: "${file}.csv",
[3c40dc2a]	487	exclusionValues: '',
	488	displayTableFlag: false,
	489	inclusionFlag: 'OFF',
	490	url: ''
[cdcd53dc]	491	]] : [];
[8d63649]	492
	493	echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
	494	dir("${BuildDir}/benchmark/") {
	495	plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
	496	csvSeries: series,
[490cb3c]	497	group: "${group.name}",
[3c40dc2a]	498	title: "${title}",
	499	style: 'lineSimple',
	500	exclZero: false,
	501	keepRecords: false,
[3221a2b]	502	logarithmic: !relative && group.log,
[3c40dc2a]	503	numBuilds: '120',
	504	useDescr: true,
[490cb3c]	505	yaxis: group.unit,
[3c40dc2a]	506	yaxisMaximum: '',
	507	yaxisMinimum: ''
[df57a84]	508	}
[8110bc3]	509	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: