Context Navigation

source: Jenkinsfile @ 3dcd11ba

ADTast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum

Last change on this file since 3dcd11ba was bd50205, checked in by Thierry Delisle <tdelisle@…>, 3 years ago
Jenkins email must run inside node
Property mode set to `100644`
File size: 15.1 KB

Rev	Line
[a63ad80]	1	#!groovy
	2
[7a230fd]	3	import groovy.transform.Field
	4
[29f4fe62]	5	//===========================================================================================================
[9beae23]	6	// Main loop of the compilation
[29f4fe62]	7	//===========================================================================================================
[56a9ce6]	8
[d3b95f1]	9	// Globals
[e018546]	10	BuildDir = null
	11	SrcDir = null
[d3b95f1]	12	Settings = null
	13	Tools = null
[4f9e706]	14
[d3b95f1]	15	// Local variables
	16	def err = null
	17	def log_needed = false
[5b8413b4]	18
[d3b95f1]	19	currentBuild.result = "SUCCESS"
[4f9e706]	20
[d3b95f1]	21	try {
[47138ee]	22	node {
	23	//Wrap build to add timestamp to command line
	24	wrap([$class: 'TimestamperBuildWrapper']) {
[85142648]	25	Settings = prepare_build()
[d3b95f1]	26	}
[47138ee]	27	}
[952ee7a]	28
[47138ee]	29	node(Settings.Architecture.node) {
	30	//Wrap build to add timestamp to command line
	31	wrap([$class: 'TimestamperBuildWrapper']) {
[d3b95f1]	32	BuildDir = pwd tmp: true
	33	SrcDir = pwd tmp: false
	34	currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
[7aebc62]	35
[d3b95f1]	36	Tools.Clean()
[fde808df]	37
[d3b95f1]	38	Tools.Checkout()
[95fdb0a]	39
[d3b95f1]	40	build()
[95fdb0a]	41
[d3b95f1]	42	test()
[95fdb0a]	43
[d3b95f1]	44	benchmark()
[7359098]	45
[d3b95f1]	46	build_doc()
[9beae23]	47
[d3b95f1]	48	publish()
[9beae23]	49	}
[738cf8f]	50	}
[d3b95f1]	51	}
[738cf8f]	52
[d3b95f1]	53	//If an exception is caught we need to change the status and remember to
	54	//attach the build log to the email
	55	catch (Exception caughtError) {
	56	// Store the result of the build log
	57	currentBuild.result = "FAILURE"
[9dd31e7]	58
[d3b95f1]	59	// An error has occured, the build log is relevent
	60	log_needed = true
[9beae23]	61
[d3b95f1]	62	// rethrow error later
	63	err = caughtError
[852ae0ea]	64
[d3b95f1]	65	// print the error so it shows in the log
	66	echo err.toString()
	67	}
[738cf8f]	68
[d3b95f1]	69	finally {
	70	//Send email with final results if this is not a full build
	71	email(log_needed)
[9beae23]	72
[d3b95f1]	73	echo 'Build Completed'
[734891d]	74
[d3b95f1]	75	/* Must re-throw exception to propagate error */
	76	if (err) {
	77	throw err
[738cf8f]	78	}
	79	}
[29f4fe62]	80	//===========================================================================================================
[9beae23]	81	// Main compilation routines
[29f4fe62]	82	//===========================================================================================================
[95fdb0a]	83	def build() {
[e507c11]	84	debug = true
	85	release = Settings.RunAllTests \|\| Settings.RunBenchmark
[91aa5ab]	86	Tools.BuildStage('Build : configure', true) {
[dcf1979]	87	// Configure must be run inside the tree
	88	dir (SrcDir) {
	89	// Generate the necessary build files
	90	sh './autogen.sh'
	91	}
	92
[50f2cfc]	93	// Build outside of the src tree to ease cleaning
[5b8413b4]	94	dir (BuildDir) {
[8e58264]	95	//Configure the compilation (Output is not relevant)
[50f2cfc]	96	//Use the current directory as the installation target so nothing escapes the sandbox
	97	//Also specify the compiler by hand
[3fc5f010]	98	targets=""
[d4510ea]	99	if( Settings.RunAllTests \|\| Settings.RunBenchmark ) {
[6bde81d]	100	targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]	101	} else {
[6bde81d]	102	targets="--with-target-hosts='host:debug'"
[3fc5f010]	103	}
	104
[d21dd3cb]	105	ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
	106
	107	sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]	108
	109	// Configure libcfa
[e70e54e]	110	sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]	111	}
	112	}
	113
[91aa5ab]	114	Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]	115	// Build outside of the src tree to ease cleaning
	116	dir (BuildDir) {
	117	// Build driver
	118	sh 'make -j 8 --no-print-directory -C driver'
	119
	120	// Build translator
	121	sh 'make -j 8 --no-print-directory -C src'
	122	}
	123	}
[1752d0e]	124
[91aa5ab]	125	Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]	126	// Build outside of the src tree to ease cleaning
	127	dir (BuildDir) {
	128	sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
	129	}
	130	}
	131
[91aa5ab]	132	Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]	133	// Build outside of the src tree to ease cleaning
	134	dir (BuildDir) {
	135	sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]	136	}
[620dd2b]	137	}
[aa96fba]	138
[91aa5ab]	139	Tools.BuildStage('Build : install', true) {
[aa96fba]	140	// Build outside of the src tree to ease cleaning
	141	dir (BuildDir) {
	142	sh "make -j 8 --no-print-directory install"
	143	}
	144	}
[95fdb0a]	145	}
[24eecab]	146
[95fdb0a]	147	def test() {
[4c1b9ea8]	148	try {
[1baf6ed]	149	// Print potential limits before testing
	150	// in case jenkins messes with them
[a33dcd5]	151	sh 'free -h'
[1baf6ed]	152	sh 'ulimit -a'
	153
[91aa5ab]	154	Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]	155	dir (BuildDir) {
[3e93c00]	156	//Run the tests from the tests directory
[4c51aca]	157	sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]	158	}
[4c1b9ea8]	159	}
	160
[91aa5ab]	161	Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]	162	dir (BuildDir) {
	163	//Run the tests from the tests directory
[4c51aca]	164	sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
	165	sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]	166	}
[9beae23]	167	}
[620dd2b]	168	}
[4c1b9ea8]	169	catch (Exception err) {
	170	echo "Archiving core dumps"
[c95fdc9]	171	dir (BuildDir) {
[cc9ec56]	172	def exists = fileExists 'tests/crashes'
	173	if( exists ) {
[cece53c]	174	sh """${SrcDir}/tools/jenkins/archive-gen.sh"""
	175	archiveArtifacts artifacts: "tests/crashes/*/,lib/*/lib.so*,setup.sh", fingerprint: true
	176	}
[c95fdc9]	177	}
[4c1b9ea8]	178	throw err
	179	}
[95fdb0a]	180	}
	181
	182	def benchmark() {
[91aa5ab]	183	Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]	184	dir (BuildDir) {
[0dc3ac3]	185	//Append bench results
[c6f1f3e]	186	sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]	187	}
[620dd2b]	188	}
[9beae23]	189	}
[efd60d67]	190
[95fdb0a]	191	def build_doc() {
[91aa5ab]	192	Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]	193	dir ('doc/user') {
	194	make_doc()
	195	}
	196
	197	dir ('doc/refrat') {
	198	make_doc()
	199	}
[620dd2b]	200	}
[9beae23]	201	}
	202
[95fdb0a]	203	def publish() {
[91aa5ab]	204	Tools.BuildStage('Publish', true) {
[95fdb0a]	205
[1b3eef8]	206	if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]	207
[3221a2b]	208	def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
	209	def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]	210
[3898392]	211	//Then publish the results
[13d2dac]	212	do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
	213	do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
	214	do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
	215	do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
	216	do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
	217	do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
	218	do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
	219	do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]	220	}
[95fdb0a]	221	}
	222
[29f4fe62]	223	//===========================================================================================================
	224	//Routine responsible of sending the email notification once the build is completed
	225	//===========================================================================================================
[a336d46]	226	//Standard build email notification
[13c98a4]	227	def email(boolean log) {
[bd50205]	228	node {
	229	//Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
	230	//Configurations for email format
	231	echo 'Notifying users of result'
	232
	233	def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
	234	def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
	235	def email_body = """<p>This is an automated email from the Jenkins build machine. It was
	236	generated because of a git hooks/post-receive script following
	237	a ref change which was pushed to the C\u2200 repository.</p>
	238	""" + Tools.GitLogMessage()
	239
	240	def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
	241
	242	if( Settings && !Settings.Silent ) {
	243	//send email notification
	244	emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
	245	} else {
	246	echo "Would send email to: ${email_to}"
	247	echo "With title: ${email_subject}"
	248	echo "Content: \n${email_body}"
	249	}
[094a42c]	250	}
[e8a22a7]	251	}
[5b8413b4]	252
	253	//===========================================================================================================
	254	// Helper classes/variables/routines
	255	//===========================================================================================================
	256	//Description of a compiler (Must be serializable since pipelines are persistent)
	257	class CC_Desc implements Serializable {
[93fe3154]	258	public String name
	259	public String CXX
[6ebc13f]	260	public String CC
[bf22bc6]	261	public String lto
[93fe3154]	262
[bf22bc6]	263	CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]	264	this.name = name
	265	this.CXX = CXX
[bf22bc6]	266	this.CC = CC
	267	this.lto = lto
[5b8413b4]	268	}
	269	}
	270
	271	//Description of an architecture (Must be serializable since pipelines are persistent)
	272	class Arch_Desc implements Serializable {
	273	public String name
	274	public String flags
[5307c33]	275	public String node
[5b8413b4]	276
[5307c33]	277	Arch_Desc(String name, String flags, String node) {
[5b8413b4]	278	this.name = name
	279	this.flags = flags
[5307c33]	280	this.node = node
[5b8413b4]	281	}
	282	}
	283
	284	class BuildSettings implements Serializable {
	285	public final CC_Desc Compiler
	286	public final Arch_Desc Architecture
[d21dd3cb]	287	public final Boolean NewAST
[5b8413b4]	288	public final Boolean RunAllTests
	289	public final Boolean RunBenchmark
	290	public final Boolean BuildDocumentation
	291	public final Boolean Publish
	292	public final Boolean Silent
	293	public final Boolean IsSandbox
	294	public final String DescLong
	295	public final String DescShort
	296
[a336d46]	297	public String GitNewRef
	298	public String GitOldRef
	299
[490cb3c]	300	BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]	301	switch( param.Compiler ) {
[099f5bd]	302	case 'gcc-9':
[bf22bc6]	303	this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]	304	break
	305	case 'gcc-8':
[bf22bc6]	306	this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]	307	break
	308	case 'gcc-7':
[bf22bc6]	309	this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]	310	break
[5b8413b4]	311	case 'gcc-6':
[bf22bc6]	312	this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]	313	break
	314	case 'gcc-5':
[bf22bc6]	315	this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]	316	break
	317	case 'gcc-4.9':
[bf22bc6]	318	this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]	319	break
	320	case 'clang':
[391c065]	321	this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]	322	break
	323	default :
	324	error "Unhandled compiler : ${cc}"
	325	}
	326
	327	switch( param.Architecture ) {
	328	case 'x64':
[a3e8281]	329	this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]	330	break
	331	case 'x86':
[a3e8281]	332	this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]	333	break
	334	default :
	335	error "Unhandled architecture : ${arch}"
	336	}
	337
[f95e8f0]	338	this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]	339	this.NewAST = param.NewAST
[5b8413b4]	340	this.RunAllTests = param.RunAllTests
[7a230fd]	341	this.RunBenchmark = param.RunBenchmark
[5b8413b4]	342	this.BuildDocumentation = param.BuildDocumentation
[7a230fd]	343	this.Publish = param.Publish
[5b8413b4]	344	this.Silent = param.Silent
	345
	346	def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]	347	this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]	348
[fe3d9ab]	349	final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]	350	this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[85799aa]	351	AST Version : ${ ast.toString() }
[5b8413b4]	352	Architecture : ${ this.Architecture.name }
	353	Arc Flags : ${ this.Architecture.flags }
	354	Run All Tests : ${ this.RunAllTests.toString() }
	355	Run Benchmark : ${ this.RunBenchmark.toString() }
	356	Build Documentation : ${ this.BuildDocumentation.toString() }
	357	Publish : ${ this.Publish.toString() }
	358	Silent : ${ this.Silent.toString() }
	359	"""
[a336d46]	360
	361	this.GitNewRef = ''
	362	this.GitOldRef = ''
[5b8413b4]	363	}
	364	}
	365
[490cb3c]	366	class PlotGroup implements Serializable {
	367	public String name
	368	public String unit
	369	public boolean log
	370
	371	PlotGroup(String name, String unit, boolean log) {
	372	this.name = name
	373	this.unit = unit
	374	this.log = log
	375	}
	376	}
	377
[5b8413b4]	378	def prepare_build() {
	379	// prepare the properties
	380	properties ([ \
[62f96ae]	381	buildDiscarder(logRotator( \
	382	artifactDaysToKeepStr: '', \
	383	artifactNumToKeepStr: '', \
	384	daysToKeepStr: '730', \
	385	numToKeepStr: '1000' \
	386	)), \
[5b8413b4]	387	[$class: 'ParametersDefinitionProperty', \
	388	parameterDefinitions: [ \
	389	[$class: 'ChoiceParameterDefinition', \
	390	description: 'Which compiler to use', \
	391	name: 'Compiler', \
[c09ae73]	392	choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]	393	defaultValue: 'gcc-8', \
[5b8413b4]	394	], \
	395	[$class: 'ChoiceParameterDefinition', \
	396	description: 'The target architecture', \
	397	name: 'Architecture', \
	398	choices: 'x64\nx86', \
	399	defaultValue: 'x64', \
	400	], \
[d21dd3cb]	401	[$class: 'BooleanParameterDefinition', \
	402	description: 'If true, build compiler using new AST', \
	403	name: 'NewAST', \
[6a531ab]	404	defaultValue: true, \
[d21dd3cb]	405	], \
[5b8413b4]	406	[$class: 'BooleanParameterDefinition', \
	407	description: 'If false, only the quick test suite is ran', \
	408	name: 'RunAllTests', \
	409	defaultValue: false, \
[fe3d9ab]	410	], \
[5b8413b4]	411	[$class: 'BooleanParameterDefinition', \
	412	description: 'If true, jenkins also runs benchmarks', \
	413	name: 'RunBenchmark', \
	414	defaultValue: false, \
	415	], \
	416	[$class: 'BooleanParameterDefinition', \
	417	description: 'If true, jenkins also builds documentation', \
	418	name: 'BuildDocumentation', \
	419	defaultValue: true, \
	420	], \
	421	[$class: 'BooleanParameterDefinition', \
	422	description: 'If true, jenkins also publishes results', \
	423	name: 'Publish', \
	424	defaultValue: false, \
	425	], \
	426	[$class: 'BooleanParameterDefinition', \
	427	description: 'If true, jenkins will not send emails', \
	428	name: 'Silent', \
	429	defaultValue: false, \
	430	], \
	431	],
	432	]])
	433
[bd8dca2]	434	// It's unfortunate but it looks like we need to checkout the entire repo just to get
	435	// - the pretty git printer
	436	// - Jenkins.tools
[4c55047]	437	checkout scm
[2407853]	438
[1483a16]	439	Tools = load "Jenkins/tools.groovy"
[bd8dca2]	440
[490cb3c]	441	final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]	442
	443	currentBuild.description = settings.DescShort
	444	echo settings.DescLong
	445
	446	return settings
	447	}
	448
	449	def make_doc() {
	450	def err = null
	451	try {
	452	sh 'make clean > /dev/null'
	453	sh 'make > /dev/null 2>&1'
	454	}
	455	catch (Exception caughtError) {
	456	err = caughtError //rethrow error later
[65f4a51]	457	sh 'cat build/*.log'
[5b8413b4]	458	}
	459	finally {
	460	if (err) throw err // Must re-throw exception to propagate error
	461	}
[a2a0065]	462	}
	463
[3221a2b]	464	def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]	465
[1b3eef8]	466	if(new_data) {
	467	echo "Publishing new data"
	468	}
	469
[cdcd53dc]	470	def series = new_data ? [[
[df57a84]	471	file: "${file}.csv",
[3c40dc2a]	472	exclusionValues: '',
	473	displayTableFlag: false,
	474	inclusionFlag: 'OFF',
	475	url: ''
[cdcd53dc]	476	]] : [];
[8d63649]	477
	478	echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
	479	dir("${BuildDir}/benchmark/") {
	480	plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
	481	csvSeries: series,
[490cb3c]	482	group: "${group.name}",
[3c40dc2a]	483	title: "${title}",
	484	style: 'lineSimple',
	485	exclZero: false,
	486	keepRecords: false,
[3221a2b]	487	logarithmic: !relative && group.log,
[3c40dc2a]	488	numBuilds: '120',
	489	useDescr: true,
[490cb3c]	490	yaxis: group.unit,
[3c40dc2a]	491	yaxisMaximum: '',
	492	yaxisMinimum: ''
[df57a84]	493	}
	494	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: