source: doc/theses/mike_brooks_MMath/benchmarks/list/Makefile@ 5ba1356

# For correctness, see test-correctness.sh.
# For performance, example:
#       pushd ~/cfax
#       . ~/setcfa build-fast
#       popd
#       make perfprogs CFA=$cfa -j8 MODE=performance
#       make results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=common5
#       cp results-latest.csv results-baseline.csv
# For performance, see also: `results-smoketest.csv`, `make thesis`, and more such applied targets at the end

CFA ?= cfa
CC ?= gcc-11
CXX ?= g++-11
UXX ?=  ~/u++/u++-7.0.0/bin/u++

MODE?=performance
EXTRA_COMP_FLAGS?=
RUN_NUM_REPS?=3
RUN_DATA_SIZE_MODE?=none
RUN_DURATION_SEC?=5
RUN_TASKSET_CPULIST?=6

LLHEAP_LIB=../llheap/libllheap.so

ifeq "$(MODE)" "performance"
PERFFLAGS_CFA = -DNDEBUG -O3 -nodebug
PERFFLAGS_CC  = -DNDEBUG -O3
else ifeq "$(MODE)" "correctness"
PERFFLAGS_CFA = -O0 -g -nodebug -D__CFA_DEBUG__ # shortcut for not also building debug cfa
PERFFLAGS_CC  = -O0 -g
else
$(error Bad MODE ($(MODE)); should be performance or correctness)
endif

PERFFLAGS_CXX = $(PERFFLAGS_CC)
PERFFLAGS_UXX = $(PERFFLAGS_CFA)

CFLAGS=$(PERFFLAGS_CC) $(EXTRA_COMP_FLAGS)

SHELL = /usr/bin/bash

# function: project an element from a filename that contains a delimited tuple
# (call proj,-,a-b-c.hfa,3)
# is
# c
define proj
$(word $(3),$(subst $(1), ,$(basename $(2))))
endef

# functions: cross two lists, adding given delimiter between
# (call cross,-,a b c,1 2)
# is
# a-1 a-2 b-1 b-2 c-1 c-2
define cross
$(foreach x,$(2),$(foreach xs,$(3),$(x)$(1)$(xs)))
endef
define cross3
$(call cross,$(1),$(2),$(call cross,$(1),$(3),$(4)))
endef
define cross4
$(call cross,$(1),$(2),$(call cross3,$(1),$(3),$(4),$(5)))
endef
define cross5
$(call cross,$(1),$(2),$(call cross4,$(1),$(3),$(4),$(5),$(6)))
endef

OP_MOVEMENTS?=stack queue
OP_POLARITIES?=insfirst inslast
OP_ACCESSORS?=allhead inselem remelem
RUN_INTERLEAVE_PCTS?=0


FX_SOLUTIONS?=lq-tailq lq-list cfa-cfa cfa-mandHead cfa-noListed cfa-noIter cfa-likeLq cfa-strip upp-upp cpp-stlref

OPS=$(call cross3,-,$(OP_MOVEMENTS),$(OP_POLARITIES),$(OP_ACCESSORS))
FXS=$(FX_SOLUTIONS)

all : perfprogs results-latest.csv

# Want to add functional dependency:
# if current FX_SOLUTION is lq-list then
# current OP_MOVEMENT must be stack and
# current OP_POLARITY must be insfirst
LQ_LIST_INCOMPAT_OP_MOVEMENTS=$(filter-out stack,$(OP_MOVEMENTS))
LQ_LIST_INCOMPAT_OP_POLARITIES=$(filter-out insfirst,$(OP_POLARITIES))
LQ_LIST_INCOMPAT_OPS=$(call cross3,-,$(LQ_LIST_INCOMPAT_OP_MOVEMENTS),$(OP_POLARITIES),$(OP_ACCESSORS)) \
                     $(call cross3,-,$(OP_MOVEMENTS),$(LQ_LIST_INCOMPAT_OP_POLARITIES),$(OP_ACCESSORS))
INCOMPAT=$(call cross,--,lq-list,$(LQ_LIST_INCOMPAT_OPS))
define filterFds
$(filter-out $(INCOMPAT),$(1))
endef


CORES_FULL=$(call cross,--,$(FXS),$(OPS))                # lq-tailq--stack-inslast-allhead
CORES=$(call filterFds,$(CORES_FULL))                    # lq-tailq--stack-inslast-allhead

.SECONDEXPANSION: # allow $$ expansion for dynamic dependencies


PERFPROGS=$(call cross,--,perfexp,$(CORES))

perfprogs : $(PERFPROGS)

perfexp--% driver--%.o result--%.1csv : FX=$(call proj,--,$@,2)
perfexp--% driver--%.o result--%.1csv : FX_COARSE=$(call proj,-,$(FX),1)
perfexp--% driver--%.o result--%.1csv : OP=$(call proj,--,$@,3)
perfexp--% driver--%.o result--%.1csv : OP_MOVEMENT=$(call proj,-,$(OP),1)
perfexp--% driver--%.o result--%.1csv : OP_POLARITY=$(call proj,-,$(OP),2)
perfexp--% driver--%.o result--%.1csv : OP_ACCESSOR=$(call proj,-,$(OP),3)
perfexp--% driver--%.o result--%.1csv : OP_DEFINES=-DOP_MOVEMENT=$(OP_MOVEMENT) -DOP_POLARITY=$(OP_POLARITY) -DOP_ACCESSOR=$(OP_ACCESSOR)

perfexp--cfa-% driver--cfa-%.o : COMPILER=$(CFA) $(PERFFLAGS_CFA)
perfexp--lq-%  driver--lq-%.o  : COMPILER=$(CC)  $(PERFFLAGS_CC)
perfexp--cpp-% driver--cpp-%.o : COMPILER=$(CXX) $(PERFFLAGS_CXX)
perfexp--upp-% driver--upp-%.o : COMPILER=$(UXX) $(PERFFLAGS_UXX)
perfexp--%     driver--%.o     : COMPILER=NO-COMPILER-FOR-$(FX_COARSE)

USE_LLHEAP=no
perfexp--cpp-% driver--cpp-%.o : USE_LLHEAP=yes
perfexp--cpp-%AllocGlib driver--cpp-%AllocGlib.o : USE_LLHEAP=no

LLHEAP_LIB_COND=$(if $(filter $(USE_LLHEAP),yes),$(LLHEAP_LIB))
LLHEAP_PRELOAD_COND=$(if $(filter $(USE_LLHEAP),yes),LD_PRELOAD=$(LLHEAP_LIB))

$(dir $(LLHEAP_LIB)) :
        $(error Manual action required: clone llheap into $@)

$(LLHEAP_LIB) : $$(dir $$@)
        $(MAKE) -C $< $(notdir $@)

# Without this %.d rule, ordinary make runs have noise about the recipe for driver--%.o being ill-formed when called on a *.d.
# https://stackoverflow.com/questions/3714041/why-does-this-makefile-execute-a-target-on-make-clean
# Whatever you -include gets called as a target first.
# One such example is driver--upp-upp--stack-insfirst-allhead.d
# Without this %.d rule, `make make driver--upp-upp--stack-insfirst-allhead.d` leads to the rule for driver--%.o firing.
# Though my dumb human eyes don't see the pattern as matching.
%.d:
        @touch $@

ifeq "$(MAKE_DISABLE_OBSERVATION)" "yes"
OBSERVATION_COMP_FLAG= -DDISABLE_OBSERVATION
OBSERVATION_OBJ_COND=
else
OBSERVATION_COMP_FLAG=
OBSERVATION_OBJ_COND= observation.o
endif

perfexp--% : $$(LLHEAP_LIB_COND) driver--%.o $(OBSERVATION_OBJ_COND)
        $(COMPILER) $(EXTRA_COMP_FLAGS) $^ -o $@

driver--%.o : driver.c
        $(COMPILER) $(EXTRA_COMP_FLAGS) $(OBSERVATION_COMP_FLAG) -c $< $(OP_DEFINES) -include op-$(OP).h -include fx-$(FX).h -o $@ -MMD 


# troubleshooting, e.g. `make echo_DEMOS` runs `echo $(DEMOS)`
echo_% :
        @echo '$($(@:echo_%=%))'


ifeq "$(RUN_DATA_SIZE_MODE)" "manual"
        ifeq "$(RUN_DATA_SIZES)" ""
        $(error RUN_DATA_SIZE_MODE is manual but RUN_DATA_SIZES not given)
        endif
endif

RUN_DATA_SIZES = \
$(if $(filter $(RUN_DATA_SIZE_MODE),common5), \
  7-1000000 \
  71-100000 \
  809-10000 \
  9051-1000 \
  72421-100 \
, $(if $(filter $(RUN_DATA_SIZE_MODE),thorough), \
  1-1000000 \
  2-1000000 \
  3-1000000 \
  5-1000000 \
  7-1000000 \
  11-100000 \
  13-100000 \
  19-100000 \
  29-100000 \
  37-100000 \
  53-100000 \
  71-100000 \
  101-10000 \
  149-10000 \
  211-10000 \
  283-10000 \
  401-10000 \
  569-10000 \
  809-10000 \
  1151-1000 \
  1601-1000 \
  2267-1000 \
  3203-1000 \
  4547-1000 \
  6473-1000 \
  9051-1000 \
  12809-100 \
  18119-100 \
  25601-100 \
  36209-100 \
  51203-100 \
  72421-100 \
  102407-10 \
  144817-10 \
  204803-10 \
  289637-10 \
  409609-10 \
  579263-10 \
  819229-10 \
  1158613-1 \
  1638431-1 \
  2317057-1 \
  3276803-1 \
  4634111-1 \
  6553621-1 \
  9268211-1 \
, $(if $(filter $(RUN_DATA_SIZE_MODE),bignquick), \
  3-1000000 \
  29-100000 \
  283-10000 \
  3203-1000 \
  25601-100 \
  289637-10 \
  1000000-1 \
  3276803-1 \
  10000000-1 \
, $(if $(filter $(RUN_DATA_SIZE_MODE),bignthorough), \
  1-1000000 \
  3-1000000 \
  7-1000000 \
  13-100000 \
  29-100000 \
  53-100000 \
  101-10000 \
  211-10000 \
  401-10000 \
  809-10000 \
  1601-1000 \
  3203-1000 \
  6473-1000 \
  12809-100 \
  25601-100 \
  51203-100 \
  102407-10 \
  204803-10 \
  409609-10 \
  819229-10 \
  1638431-1 \
  3276803-1 \
  6553621-1 \
  12809000-1 \
  25601000-1 \
  51203000-1 \
  102407000-1 \
  204803000-1 \
  409609000-1 \
, $(if $(filter $(RUN_DATA_SIZE_MODE),moderate), \
  1-1000000 \
  3-1000000 \
  7-1000000 \
  13-100000 \
  29-100000 \
  53-100000 \
  101-10000 \
  211-10000 \
  401-10000 \
  1601-1000 \
  6473-1000 \
  25601-100 \
, $(if $(filter $(RUN_DATA_SIZE_MODE),sweetspot), \
  1-1000000 \
  2-1000000 \
  3-1000000 \
  5-1000000 \
  7-1000000 \
  11-100000 \
  13-100000 \
  19-100000 \
  29-100000 \
  37-100000 \
  53-100000 \
  71-100000 \
  101-10000 \
  149-10000 \
, $(if $(filter $(RUN_DATA_SIZE_MODE),peter), \
  4-1000000 \
  8-1000000 \
  16-100000 \
  32-100000 \
  64-100000 \
  128-10000 \
  256-10000 \
  512-10000 \
  1024-1000 \
  2048-1000 \
  4096-1000 \
  8192-1000 \
  16384-100 \
  32768-100 \
  65536-100 \
  131072-10 \
  262144-10 \
  524288-10 \
  1048576-1 \
  2097152-1 \
  4194304-1 \
  8388608-1 \
  16777216-1 \
, $(if $(filter $(RUN_DATA_SIZE_MODE),none), \
, $(error Bad RUN_DATA_SIZE_MODE ($(RUN_DATA_SIZE_MODE)); see list of accepted values in Makefile's RUN_DATA_SIZES defimition) \
))))))))

RUN_REP_IDS=$(shell echo {1..$(RUN_NUM_REPS)})              # 1 2 3
RUN_REP_EXTS=$(call cross3,,run,$(RUN_REP_IDS),.1csv)       # run1.1csv run2.1csv run3.1csv

RUN_LAUNCHES=$(call cross3,--,$(RUN_DATA_SIZES),$(RUN_INTERLEAVE_PCTS),$(RUN_REP_EXTS))


RESULT1S=$(call cross,.,$(CORES),$(RUN_LAUNCHES))   # lq-tailq--stack-inslast-allhead.run2.1csv


RESULT1S_SHUFD=$(shell shuf -e $(RESULT1S))

%.1csv : CORE=$(basename $(basename $@))
%.1csv : LAUNCH=$(subst .,,$(suffix $(basename $@)))
%.1csv : SIZING=$(call proj,--,$(LAUNCH),1)
%.1csv : NUMNODES=$(call proj,-,$(SIZING),1)
%.1csv : CHECKDONE=$(call proj,-,$(SIZING),2)
%.1csv : RUN_INTERLEAVE_PCT=$(call proj,--,$(LAUNCH),2)
%.1csv : RUN_INTERLEAVE_FRAC=$(shell echo "scale=2; $(RUN_INTERLEAVE_PCT) / 100" | bc)
%.1csv : REP_ID=$(subst run,,$(call proj,--,$(LAUNCH),3))
%.1csv : SEED=$(REP_ID)
%.1csv : RUN_ARGS=$(if $(filter none,$(SIZING)),,$(RUN_DURATION_SEC) $(CHECKDONE) $(NUMNODES) -1 $(SEED) $(RUN_INTERLEAVE_FRAC))  # use REP_ID as seed
%.1csv : REP_TIME=$(shell date '+%F %H:%M:%S')
%.1csv : FX=$(call proj,--,$(CORE),1)
%.1csv : IS_CPP=$(filter cpp-%,$(FX))
%.1csv : IS_NOT_EXPLICIT_GLIB=$(filter-out %AllocGlib,$(FX))
%.1csv : USE_LLHEAP=$(if $(and $(IS_CPP),$(IS_NOT_EXPLICIT_GLIB)),yes,no)
%.1csv : perfprogs FORCE
# @echo $(FX) $(IS_CPP) $(IS_NOT_EXPLICIT_GLIB) $(USE_LLHEAP) $(LLHEAP_PRELOAD_COND)
        taskset --cpu-list $(RUN_TASKSET_CPULIST) sh -c '$(LLHEAP_PRELOAD_COND) ./perfexp--$(CORE) $(RUN_ARGS)' | xargs -n 1 printf '%s,%s,%s,%s\n' "$(REP_TIME)" "$(REP_ID)" "$(RUN_ARGS)" | tee -a $(RESULT)

# := for evaluate once and save
# =  for letting cmdline override
BATCHTIME:=$(shell date '+%F--%H-%M-%S')
RESULT=results--$(BATCHTIME).csv

sub_make=$(MAKE) --no-print-directory $(1); cat $(1) >> $(2); rm $(1);


export RESULT1S_SHUFD    # used by sh loop in $(RESULT) recipe
export RUN_DURATION_SEC  # used by sub make; occurs free in %.1csv's variable bindings (not extracted from target name)


NTESTS=$(words $(RESULT1S_SHUFD))
RUNDUR = $(shell expr $(NTESTS) \* $(RUN_DURATION_SEC))
ETA = $(shell date -d "+$(RUNDUR) seconds" +"%H:%M:%S")

$(RESULT) :
        @echo running $(NTESTS) tests, eta $(ETA)
        for r in $$RESULT1S_SHUFD; do $(MAKE) --no-print-directory RESULT=$(RESULT) $$r; done

results-latest.csv : $(RESULT)
        rm -f $@
        ln -s $< $@



clean :
        rm -f *.o *.d perfexp--*

# The FORCE business means any target that mentions it is also phony
# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html says: To always rebuild a pattern rule consider using a “force target” (see [https://www.gnu.org/software/make/manual/html_node/Force-Targets.html]).
.PHONY: all perfprogs results-latest.csv clean
FORCE:

.PRECIOUS: result--%.1csv driver--%.o perfexp--% %.o

-include *.d

results-general.csv: FORCE cleanbuild
        $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS=remelem RUN_INTERLEAVE_PCTS='0 50' FX_SOLUTIONS='lq-tailq lq-list cfa-cfa cfa-mandHead'
        cat results-latest.csv >> $@
        $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS='allhead inselem' RUN_INTERLEAVE_PCTS='0' FX_SOLUTIONS='lq-tailq lq-list cfa-cfa cfa-mandHead'
        cat results-latest.csv >> $@
        $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS=remelem RUN_INTERLEAVE_PCTS='0 50' FX_SOLUTIONS='cfa-noListed cfa-noIter cfa-likeLq'
        cat results-latest.csv >> $@
        $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS='allhead inselem' RUN_INTERLEAVE_PCTS='0' FX_SOLUTIONS='cfa-noListed cfa-noIter cfa-likeLq'
        cat results-latest.csv >> $@
        $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS=remelem RUN_INTERLEAVE_PCTS='0 50' FX_SOLUTIONS='cfa-strip upp-upp cpp-stlref'
        cat results-latest.csv >> $@
        $(MAKE) results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=sweetspot OP_ACCESSORS='allhead inselem' RUN_INTERLEAVE_PCTS='0' FX_SOLUTIONS='cfa-strip upp-upp cpp-stlref'
        cat results-latest.csv >> $@

results-zoomout-noshuf.csv: EXTRA_COMP_FLAGS+= -DTINY_USER_ITEMS
results-zoomout-noshuf.csv: FORCE cleanbuild
        make results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=bignthorough OPS=stack-insfirst-allhead FX_SOLUTIONS='lq-tailq cfa-cfa upp-upp cpp-stlref cfa-strip' SEED=-1
        cp results-latest.csv $@

results-zoomout-shuf.csv: EXTRA_COMP_FLAGS+= -DTINY_USER_ITEMS
results-zoomout-shuf.csv: FORCE cleanbuild
        make results-latest.csv RUN_DURATION_SEC=5 RUN_NUM_REPS=5 RUN_DATA_SIZE_MODE=bignthorough OPS=stack-insfirst-allhead FX_SOLUTIONS='lq-tailq cfa-cfa upp-upp cpp-stlref cfa-strip'
        cp results-latest.csv $@

thesis:
        $(MAKE) results-general.csv
        $(MAKE) results-zoomout-noshuf.csv
        $(MAKE) results-zoomout-shuf.csv

results-smoketest.csv: RUN_DATA_SIZE_MODE=bignquick
results-smoketest.csv: RUN_NUM_REPS=1
results-smoketest.csv: RUN_DURATION_SEC=1
results-smoketest.csv: OP_MOVEMENTS=stack
results-smoketest.csv: OP_POLARITIES=insfirst
results-smoketest.csv: OP_ACCESSORS=allhead
results-smoketest.csv: $(RESULT)
        mv $< $@

CPUS_AVAIL=8

export RUN_DATA_SIZE_MODE
export RUN_NUM_REPS
export RUN_DURATION_SEC
export OP_MOVEMENTS
export OP_POLARITIES
export OP_ACCESSORS
export EXTRA_COMP_FLAGS
export FX_SOLUTIONS
export MAKE_DISABLE_OBSERVATION

cleanbuild: FORCE clean
        $(MAKE) -j $(CPUS_AVAIL) perfprogs

# equivalents to peter's tests

peter-all:
        $(MAKE) results-peter-linear.csv
        $(MAKE) results-peter-random2.csv
        $(MAKE) results-peter-removeHere.csv

# debatable FX_SOLUTIONS using cfa-cfa or cfa-mandhead
# cfa-cfa faster on allhead (so I thought from thesis analysis; doubting that now under minimal harness), slower on remelem
# cfa-mandhead IS what Peter's using
results-peter-%.csv: RUN_DATA_SIZE_MODE=peter
results-peter-%.csv: RUN_NUM_REPS=3
results-peter-%.csv: RUN_DURATION_SEC=3
results-peter-%.csv: OP_MOVEMENTS=stack
results-peter-%.csv: OP_POLARITIES=insfirst
results-peter-%.csv: OP_ACCESSORS=allhead
results-peter-%.csv: FX_SOLUTIONS=cfa-mandHead cpp-stlref cpp-stlrefAllocGlib
results-peter-%.csv: MAKE_DISABLE_OBSERVATION=yes
results-peter-%.csv: cleanbuild $(RESULT)
        mv $(RESULT) $@

results-peter-linear.csv: EXTRA_COMP_FLAGS=-DDISABLE_INTERLEAVING -DDISABLE_SHUFFLING_INDIRECTION -DDISABLE_ITERS_AR -DTINY_USER_ITEMS

results-peter-random2.csv: EXTRA_COMP_FLAGS=-DDISABLE_INTERLEAVING -DDISABLE_ITERS_AR -DTINY_USER_ITEMS

results-peter-removeHere.csv: EXTRA_COMP_FLAGS=-DDISABLE_INTERLEAVING  -DDISABLE_SHUFFLING_INDIRECTION -DTINY_USER_ITEMS
results-peter-removeHere.csv: OP_ACCESSORS=remelem

peter-summary.txt: FORCE
        rm -f $@
        printf '#size\tmean\tstd\tmin\tmax\tntrials\tignore\tignore\n' >> $@
        echo >> $@
        for f in results-peter*.csv; do echo -------------- $$f ---------------- >> $@; python3 qplot.py $$f >> $@; done