source: doc/theses/mike_brooks_MMath/benchmarks/list/Makefile@ 1eea589f

Last change on this file since 1eea589f was 7806f91, checked in by Mike Brooks <mlbrooks@…>, 3 months ago

Add code for reproducing performance numbers in thesis draft of 16a843
  • Property mode set to 100644
File size: 14.6 KB
Line 
1# For correctness, see test-correctness.sh.
2# For performance:
3# pushd ~/cfax
4# . ~/setcfa build-fast
5# popd
6# make perfprogs CFA=$cfa -j8 MODE=performance
7# make results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=common5
8# cp results-latest.csv results-baseline.csv
9# make results-latest.csv OP_MOVEMENTS=stack OP_POLARITIES=insfirst OP_ACCESSORS=allhead RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=thorough
10# cp results-latest.csv results-sizing.csv
11# make results-latest.csv OP_MOVEMENTS=queue OP_POLARITIES=inslast OP_ACCESSORS=remelem RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=thorough
12# cp results-latest.csv results-sizing-b.csv
13# make results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=moderate FX_SOLUTIONS='lq-tailq lq-list cfa-cfa upp-upp'
14# cp results-latest.csv results-intrsv-cube.csv
15
16CFA = cfa
17CC = gcc-11
18CXX = g++-11
19UXX = ~/u++/u++-7.0.0/bin/u++
20
21MODE=performance
22EXTRA_COMP_FLAGS=
23RUN_NUM_REPS=3
24RUN_DATA_SIZE_MODE=none
25RUN_DURATION_SEC?=5
26RUN_TASKSET_CPULIST=6
27
28ifeq "$(MODE)" "performance"
29PERFFLAGS_CFA = -DNDEBUG -O3 -nodebug
30PERFFLAGS_CC = -DNDEBUG -O3
31else ifeq "$(MODE)" "correctness"
32PERFFLAGS_CFA = -O0 -g -nodebug -D__CFA_DEBUG__ # shortcut for not also building debug cfa
33PERFFLAGS_CC = -O0 -g
34else
35$(error Bad MODE ($(MODE)); should be performance or correctness)
36endif
37
38PERFFLAGS_CXX = $(PERFFLAGS_CC)
39PERFFLAGS_UXX = $(PERFFLAGS_CFA)
40
41CFLAGS=$(PERFFLAGS_CC) $(EXTRA_COMP_FLAGS)
42
43SHELL = /usr/bin/bash
44
45# function: project an element from a filename that contains a delimited tuple
46# (call proj,-,a-b-c.hfa,3)
47# is
48# c
49define proj
50$(word $(3),$(subst $(1), ,$(basename $(2))))
51endef
52
53# functions: cross two lists, adding given delimiter between
54# (call cross,-,a b c,1 2)
55# is
56# a-1 a-2 b-1 b-2 c-1 c-2
57define cross
58$(foreach x,$(2),$(foreach xs,$(3),$(x)$(1)$(xs)))
59endef
60define cross3
61$(call cross,$(1),$(2),$(call cross,$(1),$(3),$(4)))
62endef
63define cross4
64$(call cross,$(1),$(2),$(call cross3,$(1),$(3),$(4),$(5)))
65endef
66define cross5
67$(call cross,$(1),$(2),$(call cross4,$(1),$(3),$(4),$(5),$(6)))
68endef
69
70OP_MOVEMENTS=stack queue
71OP_POLARITIES=insfirst inslast
72OP_ACCESSORS=allhead inselem remelem
73RUN_INTERLEAVE_PCTS=0
74
75
76FX_SOLUTIONS=lq-tailq lq-list cfa-cfa cfa-mandHead cfa-noListed cfa-noIter cfa-likeLq cfa-strip upp-upp cpp-stlref
77
78OPS=$(call cross3,-,$(OP_MOVEMENTS),$(OP_POLARITIES),$(OP_ACCESSORS))
79FXS=$(FX_SOLUTIONS)
80
81all : perfprogs results-latest.csv
82
83# Want to add functional dependency:
84# if current FX_SOLUTION is lq-list then
85# current OP_MOVEMENT must be stack and
86# current OP_POLARITY must be insfirst
87LQ_LIST_INCOMPAT_OP_MOVEMENTS=$(filter-out stack,$(OP_MOVEMENTS))
88LQ_LIST_INCOMPAT_OP_POLARITIES=$(filter-out insfirst,$(OP_POLARITIES))
89LQ_LIST_INCOMPAT_OPS=$(call cross3,-,$(LQ_LIST_INCOMPAT_OP_MOVEMENTS),$(OP_POLARITIES),$(OP_ACCESSORS)) \
90 $(call cross3,-,$(OP_MOVEMENTS),$(LQ_LIST_INCOMPAT_OP_POLARITIES),$(OP_ACCESSORS))
91INCOMPAT=$(call cross,--,lq-list,$(LQ_LIST_INCOMPAT_OPS))
92define filterFds
93$(filter-out $(INCOMPAT),$(1))
94endef
95
96
97CORES_FULL=$(call cross,--,$(FXS),$(OPS)) # lq-tailq--stack-inslast-allhead
98CORES=$(call filterFds,$(CORES_FULL)) # lq-tailq--stack-inslast-allhead
99
100
101PERFPROGS=$(call cross,--,perfexp,$(CORES))
102
103perfprogs : $(PERFPROGS)
104
105perfexp--% driver--%.o result--%.1csv : FX=$(call proj,--,$@,2)
106perfexp--% driver--%.o result--%.1csv : FX_COARSE=$(call proj,-,$(FX),1)
107perfexp--% driver--%.o result--%.1csv : OP=$(call proj,--,$@,3)
108perfexp--% driver--%.o result--%.1csv : OP_MOVEMENT=$(call proj,-,$(OP),1)
109perfexp--% driver--%.o result--%.1csv : OP_POLARITY=$(call proj,-,$(OP),2)
110perfexp--% driver--%.o result--%.1csv : OP_ACCESSOR=$(call proj,-,$(OP),3)
111perfexp--% driver--%.o result--%.1csv : OP_DEFINES=-DOP_MOVEMENT=$(OP_MOVEMENT) -DOP_POLARITY=$(OP_POLARITY) -DOP_ACCESSOR=$(OP_ACCESSOR)
112
113perfexp--cfa-% driver--cfa-%.o : COMPILER=$(CFA) $(PERFFLAGS_CFA)
114perfexp--lq-% driver--lq-%.o : COMPILER=$(CC) $(PERFFLAGS_CC)
115perfexp--cpp-% driver--cpp-%.o : COMPILER=$(CXX) $(PERFFLAGS_CXX)
116perfexp--upp-% driver--upp-%.o : COMPILER=$(UXX) $(PERFFLAGS_UXX)
117perfexp--% driver--%.o : COMPILER=NO-COMPILER-FOR-$(FX_COARSE)
118
119# Without this %.d rule, ordinary make runs have noise about the recipe for driver--%.o being ill-formed when called on a *.d.
120# https://stackoverflow.com/questions/3714041/why-does-this-makefile-execute-a-target-on-make-clean
121# Whatever you -include gets called as a target first.
122# One such example is driver--upp-upp--stack-insfirst-allhead.d
123# Without this %.d rule, `make make driver--upp-upp--stack-insfirst-allhead.d` leads to the rule for driver--%.o firing.
124# Though my dumb human eyes don't see the pattern as matching.
125%.d:
126 @touch $@
127
128perfexp--% : driver--%.o observation.o
129 $(COMPILER) $(EXTRA_COMP_FLAGS) $^ -o $@
130
131driver--%.o : driver.c
132 $(COMPILER) $(EXTRA_COMP_FLAGS) -c $< $(OP_DEFINES) -include op-$(OP).h -include fx-$(FX).h -o $@ -MMD
133
134
135# troubleshooting, e.g. `make echo_DEMOS` runs `echo $(DEMOS)`
136echo_% :
137 @echo '$($(@:echo_%=%))'
138
139
140
141
142# ifeq "$(RUN_DATA_SIZE_MODE)" "common5"
143# RUN_DATA_SIZES=\
144# 7-1000000 \
145# 71-100000 \
146# 809-10000 \
147# 9051-1000 \
148# 72421-100
149# else ifeq "$(RUN_DATA_SIZE_MODE)" "thorough"
150# RUN_DATA_SIZES=\
151# 1-1000000 \
152# 2-1000000 \
153# 3-1000000 \
154# 5-1000000 \
155# 7-1000000 \
156# 11-100000 \
157# 13-100000 \
158# 19-100000 \
159# 29-100000 \
160# 37-100000 \
161# 53-100000 \
162# 71-100000 \
163# 101-10000 \
164# 149-10000 \
165# 211-10000 \
166# 283-10000 \
167# 401-10000 \
168# 569-10000 \
169# 809-10000 \
170# 1151-1000 \
171# 1601-1000 \
172# 2267-1000 \
173# 3203-1000 \
174# 4547-1000 \
175# 6473-1000 \
176# 9051-1000 \
177# 12809-100 \
178# 18119-100 \
179# 25601-100 \
180# 36209-100 \
181# 51203-100 \
182# 72421-100 \
183# 102407-10 \
184# 144817-10 \
185# 204803-10 \
186# 289637-10 \
187# 409609-10 \
188# 579263-10 \
189# 819229-10 \
190# 1158613-1 \
191# 1638431-1 \
192# 2317057-1 \
193# 3276803-1 \
194# 4634111-1 \
195# 6553621-1 \
196# 9268211-1
197# else ifeq "$(RUN_DATA_SIZE_MODE)" "bignquick"
198# RUN_DATA_SIZES=\
199# 3-1000000 \
200# 29-100000 \
201# 283-10000 \
202# 3203-1000 \
203# 25601-100 \
204# 289637-10 \
205# 1000000-1 \
206# 3276803-1 \
207# 10000000-1
208# else ifeq "$(RUN_DATA_SIZE_MODE)" "bignthorough"
209# RUN_DATA_SIZES=\
210# 1-1000000 \
211# 3-1000000 \
212# 7-1000000 \
213# 13-100000 \
214# 29-100000 \
215# 53-100000 \
216# 101-10000 \
217# 211-10000 \
218# 401-10000 \
219# 809-10000 \
220# 1601-1000 \
221# 3203-1000 \
222# 6473-1000 \
223# 12809-100 \
224# 25601-100 \
225# 51203-100 \
226# 102407-10 \
227# 204803-10 \
228# 409609-10 \
229# 819229-10 \
230# 1638431-1 \
231# 3276803-1 \
232# 6553621-1 \
233# 12809000-1 \
234# 25601000-1 \
235# 51203000-1 \
236# 102407000-1 \
237# 204803000-1 \
238# 409609000-1
239# else ifeq "$(RUN_DATA_SIZE_MODE)" "moderate"
240# RUN_DATA_SIZES=\
241# 1-1000000 \
242# 3-1000000 \
243# 7-1000000 \
244# 13-100000 \
245# 29-100000 \
246# 53-100000 \
247# 101-10000 \
248# 211-10000 \
249# 401-10000 \
250# 1601-1000 \
251# 6473-1000 \
252# 25601-100
253# else ifeq "$(RUN_DATA_SIZE_MODE)" "sweetspot"
254# RUN_DATA_SIZES=\
255# 1-1000000 \
256# 2-1000000 \
257# 3-1000000 \
258# 5-1000000 \
259# 7-1000000 \
260# 11-100000 \
261# 13-100000 \
262# 19-100000 \
263# 29-100000 \
264# 37-100000 \
265# 53-100000 \
266# 71-100000 \
267# 101-10000 \
268# 149-10000
269# else ifeq "$(RUN_DATA_SIZE_MODE)" "manual"
270# ifeq "$(RUN_DATA_SIZES)" ""
271# $(error RUN_DATA_SIZE_MODE is manual but RUN_DATA_SIZES not given)
272# endif
273# else ifeq "$(RUN_DATA_SIZE_MODE)" "none"
274# # Assume user manages RUN_ARGS; empty RUN_ARGS just means run with compiled-in defaults
275# RUN_DATA_SIZES=none
276# else
277# $(error Bad RUN_DATA_SIZE_MODE ($(RUN_DATA_SIZE_MODE)); should be common5, thorough or manual)
278# endif
279
280
281ifeq "$(RUN_DATA_SIZE_MODE)" "manual"
282 ifeq "$(RUN_DATA_SIZES)" ""
283 $(error RUN_DATA_SIZE_MODE is manual but RUN_DATA_SIZES not given)
284 endif
285endif
286
287RUN_DATA_SIZES = \
288$(if $(filter $(RUN_DATA_SIZE_MODE),common5), \
289 7-1000000 \
290 71-100000 \
291 809-10000 \
292 9051-1000 \
293 72421-100 \
294, $(if $(filter $(RUN_DATA_SIZE_MODE),thorough), \
295 1-1000000 \
296 2-1000000 \
297 3-1000000 \
298 5-1000000 \
299 7-1000000 \
300 11-100000 \
301 13-100000 \
302 19-100000 \
303 29-100000 \
304 37-100000 \
305 53-100000 \
306 71-100000 \
307 101-10000 \
308 149-10000 \
309 211-10000 \
310 283-10000 \
311 401-10000 \
312 569-10000 \
313 809-10000 \
314 1151-1000 \
315 1601-1000 \
316 2267-1000 \
317 3203-1000 \
318 4547-1000 \
319 6473-1000 \
320 9051-1000 \
321 12809-100 \
322 18119-100 \
323 25601-100 \
324 36209-100 \
325 51203-100 \
326 72421-100 \
327 102407-10 \
328 144817-10 \
329 204803-10 \
330 289637-10 \
331 409609-10 \
332 579263-10 \
333 819229-10 \
334 1158613-1 \
335 1638431-1 \
336 2317057-1 \
337 3276803-1 \
338 4634111-1 \
339 6553621-1 \
340 9268211-1 \
341, $(if $(filter $(RUN_DATA_SIZE_MODE),bignquick), \
342 3-1000000 \
343 29-100000 \
344 283-10000 \
345 3203-1000 \
346 25601-100 \
347 289637-10 \
348 1000000-1 \
349 3276803-1 \
350 10000000-1 \
351, $(if $(filter $(RUN_DATA_SIZE_MODE),bignthorough), \
352 1-1000000 \
353 3-1000000 \
354 7-1000000 \
355 13-100000 \
356 29-100000 \
357 53-100000 \
358 101-10000 \
359 211-10000 \
360 401-10000 \
361 809-10000 \
362 1601-1000 \
363 3203-1000 \
364 6473-1000 \
365 12809-100 \
366 25601-100 \
367 51203-100 \
368 102407-10 \
369 204803-10 \
370 409609-10 \
371 819229-10 \
372 1638431-1 \
373 3276803-1 \
374 6553621-1 \
375 12809000-1 \
376 25601000-1 \
377 51203000-1 \
378 102407000-1 \
379 204803000-1 \
380 409609000-1 \
381, $(if $(filter $(RUN_DATA_SIZE_MODE),moderate), \
382 1-1000000 \
383 3-1000000 \
384 7-1000000 \
385 13-100000 \
386 29-100000 \
387 53-100000 \
388 101-10000 \
389 211-10000 \
390 401-10000 \
391 1601-1000 \
392 6473-1000 \
393 25601-100 \
394, $(if $(filter $(RUN_DATA_SIZE_MODE),sweetspot), \
395 1-1000000 \
396 2-1000000 \
397 3-1000000 \
398 5-1000000 \
399 7-1000000 \
400 11-100000 \
401 13-100000 \
402 19-100000 \
403 29-100000 \
404 37-100000 \
405 53-100000 \
406 71-100000 \
407 101-10000 \
408 149-10000 \
409, $(if $(filter $(RUN_DATA_SIZE_MODE),none), \
410, $(error Bad RUN_DATA_SIZE_MODE ($(RUN_DATA_SIZE_MODE)); see list of accepted values in Makefile's RUN_DATA_SIZES defimition) \
411)))))))
412
413RUN_REP_IDS=$(shell echo {1..$(RUN_NUM_REPS)}) # 1 2 3
414RUN_REP_EXTS=$(call cross3,,run,$(RUN_REP_IDS),.1csv) # run1.1csv run2.1csv run3.1csv
415
416RUN_LAUNCHES=$(call cross3,--,$(RUN_DATA_SIZES),$(RUN_INTERLEAVE_PCTS),$(RUN_REP_EXTS))
417
418
419RESULT1S=$(call cross,.,$(CORES),$(RUN_LAUNCHES)) # lq-tailq--stack-inslast-allhead.run2.1csv
420
421
422RESULT1S_SHUFD=$(shell shuf -e $(RESULT1S))
423
424%.1csv : CORE=$(basename $(basename $@))
425%.1csv : LAUNCH=$(subst .,,$(suffix $(basename $@)))
426%.1csv : SIZING=$(call proj,--,$(LAUNCH),1)
427%.1csv : NUMNODES=$(call proj,-,$(SIZING),1)
428%.1csv : CHECKDONE=$(call proj,-,$(SIZING),2)
429%.1csv : RUN_INTERLEAVE_PCT=$(call proj,--,$(LAUNCH),2)
430%.1csv : RUN_INTERLEAVE_FRAC=$(shell echo "scale=2; $(RUN_INTERLEAVE_PCT) / 100" | bc)
431%.1csv : REP_ID=$(subst run,,$(call proj,--,$(LAUNCH),3))
432%.1csv : SEED=$(REP_ID)
433%.1csv : RUN_ARGS=$(if $(filter none,$(SIZING)),,$(RUN_DURATION_SEC) $(CHECKDONE) $(NUMNODES) -1 $(SEED) $(RUN_INTERLEAVE_FRAC)) # use REP_ID as seed
434%.1csv : REP_TIME=$(shell date '+%F %H:%M:%S')
435%.1csv : perfprogs FORCE
436 taskset --cpu-list $(RUN_TASKSET_CPULIST) ./perfexp--$(CORE) $(RUN_ARGS) | xargs -n 1 printf '%s,%s,%s,%s\n' "$(REP_TIME)" "$(REP_ID)" "$(RUN_ARGS)" | tee -a $(RESULT)
437
438
439BATCHTIME=$(shell date '+%F--%H-%M-%S')
440RESULT=results--$(BATCHTIME).csv
441
442sub_make=$(MAKE) --no-print-directory $(1); cat $(1) >> $(2); rm $(1);
443
444
445export RESULT1S_SHUFD # used by sh loop in $(RESULT) recipe
446export RUN_DURATION_SEC # used by sub make; occurs free in %.1csv's variable bindings (not extracted from target name)
447
448
449NTESTS=$(words $(RESULT1S_SHUFD))
450RUNDUR = $(shell expr $(NTESTS) \* $(RUN_DURATION_SEC))
451ETA = $(shell date -d "+$(RUNDUR) seconds" +"%H:%M:%S")
452
453$(RESULT) :
454 @echo running $(NTESTS) tests, eta $(ETA)
455 for r in $$RESULT1S_SHUFD; do $(MAKE) --no-print-directory RESULT=$(RESULT) $$r; done
456
457results-latest.csv : $(RESULT)
458 rm -f $@
459 ln -s $< $@
460
461
462
463clean :
464 rm -f *.o *.d perfexp--*
465
466# The FORCE business means any target that mentions it is also phony
467# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html says: To always rebuild a pattern rule consider using a “force target” (see [https://www.gnu.org/software/make/manual/html_node/Force-Targets.html]).
468.PHONY: all perfprogs results-latest.csv clean
469FORCE:
470
471.PRECIOUS: result--%.1csv driver--%.o perfexp--% %.o
472
473-include *.d
474
475results-general.csv: FORCE
476 $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS=remelem RUN_INTERLEAVE_PCTS='0 50' FX_SOLUTIONS='lq-tailq lq-list cfa-cfa cfa-mandHead'
477 cat results-latest.csv >> $@
478 $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS='allhead inselem' RUN_INTERLEAVE_PCTS='0' FX_SOLUTIONS='lq-tailq lq-list cfa-cfa cfa-mandHead'
479 cat results-latest.csv >> $@
480 $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS=remelem RUN_INTERLEAVE_PCTS='0 50' FX_SOLUTIONS='cfa-noListed cfa-noIter cfa-likeLq'
481 cat results-latest.csv >> $@
482 $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS='allhead inselem' RUN_INTERLEAVE_PCTS='0' FX_SOLUTIONS='cfa-noListed cfa-noIter cfa-likeLq'
483 cat results-latest.csv >> $@
484 $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS=remelem RUN_INTERLEAVE_PCTS='0 50' FX_SOLUTIONS='cfa-strip upp-upp cpp-stlref'
485 cat results-latest.csv >> $@
486 $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS='allhead inselem' RUN_INTERLEAVE_PCTS='0' FX_SOLUTIONS='cfa-strip upp-upp cpp-stlref'
487 cat results-latest.csv >> $@
488
489results-zoomout-noshuf.csv: FORCE
490 @echo Do we need to make for tiny user iters? If so:
491 @echo make clean
492 @echo make perfprogs CFA=$$cfa EXTRA_COMP_FLAGS=-DTINY_USER_ITEMS -j8
493 make results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=bignthorough OPS=stack-insfirst-allhead FX_SOLUTIONS='lq-tailq cfa-cfa upp-upp cpp-stlref cfa-strip' SEED=-1
494 cp results-latest.csv $@
495
496results-zoomout-shuf.csv: FORCE
497 @echo Do we need to make for tiny user iters? If so:
498 @echo make clean
499 @echo make perfprogs CFA=$$cfa EXTRA_COMP_FLAGS=-DTINY_USER_ITEMS -j8
500 make results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=bignthorough OPS=stack-insfirst-allhead FX_SOLUTIONS='lq-tailq cfa-cfa upp-upp cpp-stlref cfa-strip'
501 cp results-latest.csv $@
502
503thesis: results-general.csv results-zoomout-noshuf.csv results-zoomout-shuf.csv
504
505# matches peter's "random"
506results-smoketest.csv: RUN_DATA_SIZE_MODE=bignquick
507results-smoketest.csv: RUN_NUM_REPS=1
508results-smoketest.csv: RUN_DURATION_SEC=1
509results-smoketest.csv: OP_MOVEMENTS=stack
510results-smoketest.csv: OP_POLARITIES=insfirst
511results-smoketest.csv: OP_ACCESSORS=allhead
512results-smoketest.csv: $(RESULT)
513 mv $< $@
Note: See TracBrowser for help on using the repository browser.