Context Navigation

source: Jenkinsfile@ 47138ee

Visit:

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since 47138ee was 47138ee, checked in by Thierry Delisle <tdelisle@…>, 5 years ago
wrap apparently needs to run in a node... for some reason
Property mode set to `100644`
File size: 15.1 KB

Rev	Line
[a63ad80]	1	#!groovy
	2
[7a230fd]	3	import groovy.transform.Field
	4
[29f4fe62]	5	//===========================================================================================================
[9beae23]	6	// Main loop of the compilation
[29f4fe62]	7	//===========================================================================================================
[56a9ce6]	8
[d3b95f1]	9	// Globals
[a8d8547]	10	BuildDir = ""
	11	SrcDir = ""
[d3b95f1]	12	Settings = null
	13	Tools = null
[4f9e706]	14
[d3b95f1]	15	// Local variables
	16	def err = null
	17	def log_needed = false
[5b8413b4]	18
[d3b95f1]	19	currentBuild.result = "SUCCESS"
[4f9e706]	20
[d3b95f1]	21	try {
[47138ee]	22	node {
	23	//Wrap build to add timestamp to command line
	24	wrap([$class: 'TimestamperBuildWrapper']) {
[85142648]	25	Settings = prepare_build()
[d3b95f1]	26	}
[47138ee]	27	}
[952ee7a]	28
[47138ee]	29	node(Settings.Architecture.node) {
	30	//Wrap build to add timestamp to command line
	31	wrap([$class: 'TimestamperBuildWrapper']) {
[d3b95f1]	32	BuildDir = pwd tmp: true
	33	SrcDir = pwd tmp: false
	34	currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
[7aebc62]	35
[d3b95f1]	36	Tools.Clean()
[fde808df]	37
[d3b95f1]	38	Tools.Checkout()
[95fdb0a]	39
[d3b95f1]	40	build()
[95fdb0a]	41
[d3b95f1]	42	test()
[95fdb0a]	43
[d3b95f1]	44	benchmark()
[7359098]	45
[d3b95f1]	46	build_doc()
[9beae23]	47
[d3b95f1]	48	publish()
[9beae23]	49	}
[738cf8f]	50	}
[d3b95f1]	51	}
[738cf8f]	52
[d3b95f1]	53	//If an exception is caught we need to change the status and remember to
	54	//attach the build log to the email
	55	catch (Exception caughtError) {
	56	// Store the result of the build log
	57	currentBuild.result = "FAILURE"
[9dd31e7]	58
[d3b95f1]	59	// An error has occured, the build log is relevent
	60	log_needed = true
[9beae23]	61
[d3b95f1]	62	// rethrow error later
	63	err = caughtError
[852ae0ea]	64
[d3b95f1]	65	// print the error so it shows in the log
	66	echo err.toString()
	67	}
[738cf8f]	68
[d3b95f1]	69	finally {
	70	//Send email with final results if this is not a full build
	71	email(log_needed)
[9beae23]	72
[d3b95f1]	73	echo 'Build Completed'
[734891d]	74
[d3b95f1]	75	/* Must re-throw exception to propagate error */
	76	if (err) {
	77	throw err
[738cf8f]	78	}
	79	}
[29f4fe62]	80	//===========================================================================================================
[9beae23]	81	// Main compilation routines
[29f4fe62]	82	//===========================================================================================================
[95fdb0a]	83	def build() {
[e507c11]	84	debug = true
	85	release = Settings.RunAllTests \|\| Settings.RunBenchmark
[91aa5ab]	86	Tools.BuildStage('Build : configure', true) {
[dcf1979]	87	// Configure must be run inside the tree
	88	dir (SrcDir) {
	89	// Generate the necessary build files
	90	sh './autogen.sh'
	91	}
	92
[50f2cfc]	93	// Build outside of the src tree to ease cleaning
[5b8413b4]	94	dir (BuildDir) {
[8e58264]	95	//Configure the compilation (Output is not relevant)
[50f2cfc]	96	//Use the current directory as the installation target so nothing escapes the sandbox
	97	//Also specify the compiler by hand
[3fc5f010]	98	targets=""
[d4510ea]	99	if( Settings.RunAllTests \|\| Settings.RunBenchmark ) {
[6bde81d]	100	targets="--with-target-hosts='host:debug,host:nodebug'"
[3fc5f010]	101	} else {
[6bde81d]	102	targets="--with-target-hosts='host:debug'"
[3fc5f010]	103	}
	104
[d21dd3cb]	105	ast = Settings.NewAST ? "--enable-new-ast" : "--disable-new-ast"
	106
	107	sh "${SrcDir}/configure CXX=${Settings.Compiler.CXX} CC=${Settings.Compiler.CC} ${Settings.Architecture.flags} AR=gcc-ar RANLIB=gcc-ranlib ${targets} ${ast} --quiet --prefix=${BuildDir}"
[f253e4a]	108
	109	// Configure libcfa
[e70e54e]	110	sh 'make -j 8 --no-print-directory configure-libcfa'
[e507c11]	111	}
	112	}
	113
[91aa5ab]	114	Tools.BuildStage('Build : cfa-cpp', true) {
[e507c11]	115	// Build outside of the src tree to ease cleaning
	116	dir (BuildDir) {
	117	// Build driver
	118	sh 'make -j 8 --no-print-directory -C driver'
	119
	120	// Build translator
	121	sh 'make -j 8 --no-print-directory -C src'
	122	}
	123	}
[1752d0e]	124
[91aa5ab]	125	Tools.BuildStage('Build : libcfa(debug)', debug) {
[e507c11]	126	// Build outside of the src tree to ease cleaning
	127	dir (BuildDir) {
	128	sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-debug"
	129	}
	130	}
	131
[91aa5ab]	132	Tools.BuildStage('Build : libcfa(nodebug)', release) {
[e507c11]	133	// Build outside of the src tree to ease cleaning
	134	dir (BuildDir) {
	135	sh "make -j 8 --no-print-directory -C libcfa/${Settings.Architecture.name}-nodebug"
[50f2cfc]	136	}
[620dd2b]	137	}
[aa96fba]	138
[91aa5ab]	139	Tools.BuildStage('Build : install', true) {
[aa96fba]	140	// Build outside of the src tree to ease cleaning
	141	dir (BuildDir) {
	142	sh "make -j 8 --no-print-directory install"
	143	}
	144	}
[95fdb0a]	145	}
[24eecab]	146
[95fdb0a]	147	def test() {
[4c1b9ea8]	148	try {
[1baf6ed]	149	// Print potential limits before testing
	150	// in case jenkins messes with them
[a33dcd5]	151	sh 'free -h'
[1baf6ed]	152	sh 'ulimit -a'
	153
[91aa5ab]	154	Tools.BuildStage('Test: short', !Settings.RunAllTests) {
[4c1b9ea8]	155	dir (BuildDir) {
[3e93c00]	156	//Run the tests from the tests directory
[4c51aca]	157	sh "make --no-print-directory -C tests archiveerrors=${BuildDir}/tests/crashes/short"
[3e93c00]	158	}
[4c1b9ea8]	159	}
	160
[91aa5ab]	161	Tools.BuildStage('Test: full', Settings.RunAllTests) {
[4c1b9ea8]	162	dir (BuildDir) {
	163	//Run the tests from the tests directory
[4c51aca]	164	sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=yes archiveerrors=${BuildDir}/tests/crashes/full-debug"""
	165	sh """make --no-print-directory -C tests timeouts="--timeout=600 --global-timeout=14400" all-tests debug=no archiveerrors=${BuildDir}/tests/crashes/full-nodebug"""
[143e6f3]	166	}
[9beae23]	167	}
[620dd2b]	168	}
[4c1b9ea8]	169	catch (Exception err) {
	170	echo "Archiving core dumps"
[c95fdc9]	171	dir (BuildDir) {
[cc9ec56]	172	def exists = fileExists 'tests/crashes'
	173	if( exists ) {
[cece53c]	174	sh """${SrcDir}/tools/jenkins/archive-gen.sh"""
	175	archiveArtifacts artifacts: "tests/crashes/*/,lib/*/lib.so*,setup.sh", fingerprint: true
	176	}
[c95fdc9]	177	}
[4c1b9ea8]	178	throw err
	179	}
[95fdb0a]	180	}
	181
	182	def benchmark() {
[91aa5ab]	183	Tools.BuildStage('Benchmark', Settings.RunBenchmark) {
[5b8413b4]	184	dir (BuildDir) {
[0dc3ac3]	185	//Append bench results
[c6f1f3e]	186	sh "make --no-print-directory -C benchmark jenkins arch=${Settings.Architecture.name}"
[0dc3ac3]	187	}
[620dd2b]	188	}
[9beae23]	189	}
[efd60d67]	190
[95fdb0a]	191	def build_doc() {
[91aa5ab]	192	Tools.BuildStage('Documentation', Settings.BuildDocumentation) {
[9beae23]	193	dir ('doc/user') {
	194	make_doc()
	195	}
	196
	197	dir ('doc/refrat') {
	198	make_doc()
	199	}
[620dd2b]	200	}
[9beae23]	201	}
	202
[95fdb0a]	203	def publish() {
[91aa5ab]	204	Tools.BuildStage('Publish', true) {
[95fdb0a]	205
[1b3eef8]	206	if( Settings.Publish && !Settings.RunBenchmark ) { echo 'No results to publish!!!' }
[95fdb0a]	207
[3221a2b]	208	def groupCompile = new PlotGroup('Compilation', 'duration (s) - lower is better', true)
	209	def groupConcurrency = new PlotGroup('Concurrency', 'duration (n) - lower is better', false)
[a2a0065]	210
[3898392]	211	//Then publish the results
[13d2dac]	212	do_plot(Settings.RunBenchmark && Settings.Publish, 'compile' , groupCompile , false, 'Compilation')
	213	do_plot(Settings.RunBenchmark && Settings.Publish, 'compile.diff' , groupCompile , true , 'Compilation (relative)')
	214	do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch' , groupConcurrency, false, 'Context Switching')
	215	do_plot(Settings.RunBenchmark && Settings.Publish, 'ctxswitch.diff' , groupConcurrency, true , 'Context Switching (relative)')
	216	do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex' , groupConcurrency, false, 'Mutual Exclusion')
	217	do_plot(Settings.RunBenchmark && Settings.Publish, 'mutex.diff' , groupConcurrency, true , 'Mutual Exclusion (relative)')
	218	do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling' , groupConcurrency, false, 'Internal and External Scheduling')
	219	do_plot(Settings.RunBenchmark && Settings.Publish, 'scheduling.diff', groupConcurrency, true , 'Internal and External Scheduling (relative)')
[620dd2b]	220	}
[95fdb0a]	221	}
	222
[29f4fe62]	223	//===========================================================================================================
	224	//Routine responsible of sending the email notification once the build is completed
	225	//===========================================================================================================
[a336d46]	226	//Standard build email notification
[13c98a4]	227	def email(boolean log) {
[a336d46]	228	//Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
	229	//Configurations for email format
	230	echo 'Notifying users of result'
	231
	232	def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
	233	def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
[13c98a4]	234	def email_body = """<p>This is an automated email from the Jenkins build machine. It was
[a336d46]	235	generated because of a git hooks/post-receive script following
[986e260]	236	a ref change which was pushed to the C\u2200 repository.</p>
[6a531ab]	237	""" + Tools.GitLogMessage()
[e8a22a7]	238
[13c98a4]	239	def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
[e8a22a7]	240
[13c98a4]	241	if( Settings && !Settings.Silent ) {
[094a42c]	242	//send email notification
	243	emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
	244	} else {
	245	echo "Would send email to: ${email_to}"
	246	echo "With title: ${email_subject}"
	247	echo "Content: \n${email_body}"
	248	}
[e8a22a7]	249	}
[5b8413b4]	250
	251	//===========================================================================================================
	252	// Helper classes/variables/routines
	253	//===========================================================================================================
	254	//Description of a compiler (Must be serializable since pipelines are persistent)
	255	class CC_Desc implements Serializable {
[93fe3154]	256	public String name
	257	public String CXX
[6ebc13f]	258	public String CC
[bf22bc6]	259	public String lto
[93fe3154]	260
[bf22bc6]	261	CC_Desc(String name, String CXX, String CC, String lto) {
[93fe3154]	262	this.name = name
	263	this.CXX = CXX
[bf22bc6]	264	this.CC = CC
	265	this.lto = lto
[5b8413b4]	266	}
	267	}
	268
	269	//Description of an architecture (Must be serializable since pipelines are persistent)
	270	class Arch_Desc implements Serializable {
	271	public String name
	272	public String flags
[5307c33]	273	public String node
[5b8413b4]	274
[5307c33]	275	Arch_Desc(String name, String flags, String node) {
[5b8413b4]	276	this.name = name
	277	this.flags = flags
[5307c33]	278	this.node = node
[5b8413b4]	279	}
	280	}
	281
	282	class BuildSettings implements Serializable {
	283	public final CC_Desc Compiler
	284	public final Arch_Desc Architecture
[d21dd3cb]	285	public final Boolean NewAST
[5b8413b4]	286	public final Boolean RunAllTests
	287	public final Boolean RunBenchmark
	288	public final Boolean BuildDocumentation
	289	public final Boolean Publish
	290	public final Boolean Silent
	291	public final Boolean IsSandbox
	292	public final String DescLong
	293	public final String DescShort
	294
[a336d46]	295	public String GitNewRef
	296	public String GitOldRef
	297
[490cb3c]	298	BuildSettings(java.util.Collections$UnmodifiableMap param, String branch) {
[5b8413b4]	299	switch( param.Compiler ) {
[099f5bd]	300	case 'gcc-9':
[bf22bc6]	301	this.Compiler = new CC_Desc('gcc-9', 'g++-9', 'gcc-9', '-flto=auto')
[099f5bd]	302	break
	303	case 'gcc-8':
[bf22bc6]	304	this.Compiler = new CC_Desc('gcc-8', 'g++-8', 'gcc-8', '-flto=auto')
[099f5bd]	305	break
	306	case 'gcc-7':
[bf22bc6]	307	this.Compiler = new CC_Desc('gcc-7', 'g++-7', 'gcc-7', '-flto=auto')
[099f5bd]	308	break
[5b8413b4]	309	case 'gcc-6':
[bf22bc6]	310	this.Compiler = new CC_Desc('gcc-6', 'g++-6', 'gcc-6', '-flto=auto')
[5b8413b4]	311	break
	312	case 'gcc-5':
[bf22bc6]	313	this.Compiler = new CC_Desc('gcc-5', 'g++-5', 'gcc-5', '-flto=auto')
[5b8413b4]	314	break
	315	case 'gcc-4.9':
[bf22bc6]	316	this.Compiler = new CC_Desc('gcc-4.9', 'g++-4.9', 'gcc-4.9', '-flto=auto')
[5b8413b4]	317	break
	318	case 'clang':
[391c065]	319	this.Compiler = new CC_Desc('clang', 'clang++-10', 'gcc-9', '-flto=thin -flto-jobs=0')
[5b8413b4]	320	break
	321	default :
	322	error "Unhandled compiler : ${cc}"
	323	}
	324
	325	switch( param.Architecture ) {
	326	case 'x64':
[a3e8281]	327	this.Architecture = new Arch_Desc('x64', '--host=x86_64', 'x64')
[5b8413b4]	328	break
	329	case 'x86':
[a3e8281]	330	this.Architecture = new Arch_Desc('x86', '--host=i386', 'x86')
[5b8413b4]	331	break
	332	default :
	333	error "Unhandled architecture : ${arch}"
	334	}
	335
[f95e8f0]	336	this.IsSandbox = (branch == "jenkins-sandbox")
[d21dd3cb]	337	this.NewAST = param.NewAST
[5b8413b4]	338	this.RunAllTests = param.RunAllTests
[7a230fd]	339	this.RunBenchmark = param.RunBenchmark
[5b8413b4]	340	this.BuildDocumentation = param.BuildDocumentation
[7a230fd]	341	this.Publish = param.Publish
[5b8413b4]	342	this.Silent = param.Silent
	343
	344	def full = param.RunAllTests ? " (Full)" : ""
[490cb3c]	345	this.DescShort = "${ this.Compiler.name }:${ this.Architecture.name }${full}"
[5b8413b4]	346
[fe3d9ab]	347	final ast = this.NewAST ? "New AST" : "Old AST"
[93fe3154]	348	this.DescLong = """Compiler : ${ this.Compiler.name } (${ this.Compiler.CXX }/${ this.Compiler.CC })
[85799aa]	349	AST Version : ${ ast.toString() }
[5b8413b4]	350	Architecture : ${ this.Architecture.name }
	351	Arc Flags : ${ this.Architecture.flags }
	352	Run All Tests : ${ this.RunAllTests.toString() }
	353	Run Benchmark : ${ this.RunBenchmark.toString() }
	354	Build Documentation : ${ this.BuildDocumentation.toString() }
	355	Publish : ${ this.Publish.toString() }
	356	Silent : ${ this.Silent.toString() }
	357	"""
[a336d46]	358
	359	this.GitNewRef = ''
	360	this.GitOldRef = ''
[5b8413b4]	361	}
	362	}
	363
[490cb3c]	364	class PlotGroup implements Serializable {
	365	public String name
	366	public String unit
	367	public boolean log
	368
	369	PlotGroup(String name, String unit, boolean log) {
	370	this.name = name
	371	this.unit = unit
	372	this.log = log
	373	}
	374	}
	375
[5b8413b4]	376	def prepare_build() {
	377	// prepare the properties
	378	properties ([ \
[62f96ae]	379	buildDiscarder(logRotator( \
	380	artifactDaysToKeepStr: '', \
	381	artifactNumToKeepStr: '', \
	382	daysToKeepStr: '730', \
	383	numToKeepStr: '1000' \
	384	)), \
[5b8413b4]	385	[$class: 'ParametersDefinitionProperty', \
	386	parameterDefinitions: [ \
	387	[$class: 'ChoiceParameterDefinition', \
	388	description: 'Which compiler to use', \
	389	name: 'Compiler', \
[c09ae73]	390	choices: 'gcc-9\ngcc-8\ngcc-7\ngcc-6\ngcc-5\ngcc-4.9\nclang', \
[fe27d99]	391	defaultValue: 'gcc-8', \
[5b8413b4]	392	], \
	393	[$class: 'ChoiceParameterDefinition', \
	394	description: 'The target architecture', \
	395	name: 'Architecture', \
	396	choices: 'x64\nx86', \
	397	defaultValue: 'x64', \
	398	], \
[d21dd3cb]	399	[$class: 'BooleanParameterDefinition', \
	400	description: 'If true, build compiler using new AST', \
	401	name: 'NewAST', \
[6a531ab]	402	defaultValue: true, \
[d21dd3cb]	403	], \
[5b8413b4]	404	[$class: 'BooleanParameterDefinition', \
	405	description: 'If false, only the quick test suite is ran', \
	406	name: 'RunAllTests', \
	407	defaultValue: false, \
[fe3d9ab]	408	], \
[5b8413b4]	409	[$class: 'BooleanParameterDefinition', \
	410	description: 'If true, jenkins also runs benchmarks', \
	411	name: 'RunBenchmark', \
	412	defaultValue: false, \
	413	], \
	414	[$class: 'BooleanParameterDefinition', \
	415	description: 'If true, jenkins also builds documentation', \
	416	name: 'BuildDocumentation', \
	417	defaultValue: true, \
	418	], \
	419	[$class: 'BooleanParameterDefinition', \
	420	description: 'If true, jenkins also publishes results', \
	421	name: 'Publish', \
	422	defaultValue: false, \
	423	], \
	424	[$class: 'BooleanParameterDefinition', \
	425	description: 'If true, jenkins will not send emails', \
	426	name: 'Silent', \
	427	defaultValue: false, \
	428	], \
	429	],
	430	]])
	431
[bd8dca2]	432	// It's unfortunate but it looks like we need to checkout the entire repo just to get
	433	// - the pretty git printer
	434	// - Jenkins.tools
[4c55047]	435	checkout scm
[2407853]	436
[1483a16]	437	Tools = load "Jenkins/tools.groovy"
[bd8dca2]	438
[490cb3c]	439	final settings = new BuildSettings(params, env.BRANCH_NAME)
[5b8413b4]	440
	441	currentBuild.description = settings.DescShort
	442	echo settings.DescLong
	443
	444	return settings
	445	}
	446
	447	def make_doc() {
	448	def err = null
	449	try {
	450	sh 'make clean > /dev/null'
	451	sh 'make > /dev/null 2>&1'
	452	}
	453	catch (Exception caughtError) {
	454	err = caughtError //rethrow error later
[65f4a51]	455	sh 'cat build/*.log'
[5b8413b4]	456	}
	457	finally {
	458	if (err) throw err // Must re-throw exception to propagate error
	459	}
[a2a0065]	460	}
	461
[3221a2b]	462	def do_plot(boolean new_data, String file, PlotGroup group, boolean relative, String title) {
[8d63649]	463
[1b3eef8]	464	if(new_data) {
	465	echo "Publishing new data"
	466	}
	467
[cdcd53dc]	468	def series = new_data ? [[
[df57a84]	469	file: "${file}.csv",
[3c40dc2a]	470	exclusionValues: '',
	471	displayTableFlag: false,
	472	inclusionFlag: 'OFF',
	473	url: ''
[cdcd53dc]	474	]] : [];
[8d63649]	475
	476	echo "file is ${BuildDir}/benchmark/${file}.csv, group ${group}, title ${title}"
	477	dir("${BuildDir}/benchmark/") {
	478	plot csvFileName: "cforall-${env.BRANCH_NAME}-${file}.csv",
	479	csvSeries: series,
[490cb3c]	480	group: "${group.name}",
[3c40dc2a]	481	title: "${title}",
	482	style: 'lineSimple',
	483	exclZero: false,
	484	keepRecords: false,
[3221a2b]	485	logarithmic: !relative && group.log,
[3c40dc2a]	486	numBuilds: '120',
	487	useDescr: true,
[490cb3c]	488	yaxis: group.unit,
[3c40dc2a]	489	yaxisMaximum: '',
	490	yaxisMinimum: ''
[df57a84]	491	}
	492	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: