#!/bin/bash -

false=0; true=1

# Usage: arch [ hostname ] returns hostname, cores, startcore
#
#   Define machine architecture based on starting socket, CPUs (cores) per socket, number of
#   sockets, has hyperthreading.

start=0

arch() {
	hostname=${1:-`hostname`}			# return value
	hashyper=${true}					# assume machine has hyperthreads
	if [ "${hostname}" = "plg2" ] ; then
		startsocket=${start}
		cps=16							# coresPerSocket
		sockets=2
		hashyper=${false}				# has no hyperthreads
	elif [ "${hostname}" = "nasus" ] ; then
		startsocket=${start}
		cps=64							# coresPerSocket
		sockets=2
	elif [ "${hostname}" = "pyke" ] ; then
		startsocket=${start}
		cps=24							# coresPerSocket
		sockets=2
	elif [ "${hostname}" = "jax" ] ; then
		startsocket=${start}
		cps=24							# coresPerSocket
		sockets=4
	else
		echo "unsupported host" ${hostname}
		exit 1
	fi
	cores=$(( ${cps} * ${sockets} ))
	startcore=$(( ${startsocket} * ${cps} ))
}

# Usage: affinity (global cps, sockets, startsocket, hashyper, cores, startcore, wrap)
#   returns taskset argument
#
#   This routine assumes hyperthreading has only 2 hyperthreads per core.
#
#   If hyperthread scanning is used: processor units are assigned across the low-number hyperthreads
#   of the socket's cores. When the low-number hyperthreads are filled, the high-number hyperhtreads
#   are assigned across the socket's cores. Then the next socket is assigned.
#
#   If hyperthread wrapping is used: processor units are assigned in low/high-number pairs of
#   hyperthreads across the socket's cores. Then the next socket is assigned.

wrap=${false}							# set to control hyperthread assignment across socket cores

affinity() {
	if [ ${wrap} -eq ${true} -a ${hashyper} -eq ${false} ] ; then
		echo "architecture does not support hyperthreading for wrapping"
		exit 1
	fi
	taskset=""							# return value
	set -- $(( ${1} - 1 ))				# decrement $1
	if [ ${1} -eq 0 ] ; then taskset="${startcore}-${startcore}"; return; fi
	if [ ${1} -ge $(( ${cps} * ( ${sockets} - ${startsocket} ) * ( ${hashyper} + 1 ) )) ] ; then # error
		echo "not enough cores $(( ${cores} * ${sockets} )) for $(( ${1} + 1 )) starting at ${startcore}"
		exit 1
	fi
	if [ ${hashyper} -eq ${false} ] ; then taskset="${startcore}-$(( ${1} + ${startcore} ))"; return; fi # no hyperthreads
	start2=$(( ${startcore} + ${cores} ))
	if [ ${wrap} -eq ${true} ] ; then 	# hyperthread wrapping
		end1=$(( ${1} / 2 + ${startcore} ))
		end2=$(( ${end1} + ${cores} ))
		if [ $(( ${1} % 2 )) -eq 0 ] ; then
			end2=$(( ${end2} - 1 ))
		fi
		taskset="${startcore}-${end1},${start2}-${end2}"
	else								# hyperthread scanning
		if [ ${1} -lt ${cps} ] ; then taskset="${startcore}-$(( ${1} + ${startcore} ))"; return; fi
		filled=$(( ${1} / ( ${cps} * 2 ) * ${cps} ))
		modulus=$(( ${1} % ( ${cps} * 2 ) ))	# leftover cores added to saturated sockets
		if [ ${modulus} -gt ${cps} ] ; then
			taskset="${startcore}-$(( ${startcore} + ${filled} + ${cps} - 1 )),${start2}-$(( ${start2} + ${filled} + ${modulus} % ${cps} ))"
		else
			taskset="${startcore}-$(( ${startcore} + ${filled} + ${modulus} )),${start2}-$(( ${start2} + ${filled} - 1 ))"
		fi
	fi
}

# executor config
batch='100'
nRounds='400' #500

# matrix config
size='3072'

# repeat config
messages='100000'
n_repeats='200' #200

# balance config
nOneRounds='2000'
nMultiRounds='800'

# static config
n_static_sends='100000000'

# dynamic config
n_dynamic_sends='20000000'

numtimes=5

# bench_cores='1 2 4 8 16 24 32'
# bench_cores='1 2 4 8 16 24 32 48'
bench_cores='48'

# toggle benchmarks
executor=${false}
matrix=${false}
repeat=${false}
balance=${false}
static=${true}
dynamic=${true}

# names=('CFA-LV' 'CFA-NS' 'CFA-R')
# var_flags=('-D__STEAL=1 -DSEARCH=1' '' '-D__STEAL=1 -DRAND=1')

# names=('CFA-NS')
# var_flags=('')

names=('CFA')
var_flags=('-D__STEAL=1 -DSEARCH=1')

# names=('CFA' 'CFA-STAT')
# var_flags=('-D__STEAL=1 -DSEARCH=1' '-D__STEAL=1 -DSTATS')

# names=()
# var_flags=()

runCAF=${true}
runUCPP=${true}
runPROTO=${true}
runAKKA=${true}
# runCAF=${false}
# runUCPP=${false}
# runPROTO=${false}
# runAKKA=${false}

cfa=~/cfa-cc/driver/cfa

# Helpers to minimize code duplication

# repeats a command ${numtimes}
preprint=''
repeat_command() {
    t=1
    while [ ${t} -le ${numtimes} ] ; do
        echo -n -e ${preprint}
        "${@}"
        t=`expr ${t} + 1`
    done
}

# prints the leading info for a given run of a variant
print_header() {
    echo ${1}':'
    echo -e "proc\ttime (s)"
}

# runs the current benchmark with provided args
# only works for standard-run benchmarks (not Akka)
# must split into pre and post args to be able to supply val of p
pre_args=''
post_args=''
single_run() {
    affinity ${1}
    preprint="${1}\t"
    repeat_command taskset -c ${taskset} ./a.${hostname} ${pre_args} ${1} ${post_args}
}

# runs the current bench for all processor vals
# works for standard benchs that dont need to set a config file (not Akka or CAF)
run_bench() {
    for p in ${bench_cores} ; do
        single_run ${p}
    done
}

set_akka_parallelism() {
    sed -i "s/parallelism-min = .*/parallelism-min = ${1}/g" application.conf
    sed -i "s/parallelism-max = .*/parallelism-max = ${1}/g" application.conf
}

arch # get hostname

# set up leading info for python script
echo $numtimes
echo $bench_cores

for i in ${!names[@]}; do
        echo -n ${names[$i]}" "
done
if [ ${runCAF} -eq ${true} ] ; then
    echo -n 'CAF '
fi # done CAF
if [ ${runAKKA} -eq ${true} ] ; then
    echo -n 'Akka '
fi # done AKKA
if [ ${runUCPP} -eq ${true} ] ; then
    echo -n 'uC++ '
fi # done UCPP
if [ ${runPROTO} -eq ${true} ] ; then
    echo -n 'ProtoActor '
fi
echo ""

# done printing header info for output

# cfa flags
cfa_flags='-quiet -O3 -nodebug -DNDEBUG'

# CAF flags
prefixCAF='-O3 -Wall -std=c++17 -I/home/pabuhr/software/nasus/Actors/experiments/CAF/actor-framework/libcaf_core -I/home/pabuhr/software/nasus/Actors/experiments/CAF/actor-framework/libcaf_core/caf -I/home/pabuhr/software/nasus/Actors/experiments/CAF/actor-framework/build/libcaf_core -L/home/pabuhr/software/nasus/Actors/experiments/CAF/actor-framework/build/libcaf_core -L/home/pabuhr/software/nasus/Actors/experiments/CAF/actor-framework/build/libcaf_io'
suffixCAF='-lcaf_io -lcaf_core -Wl,-rpath=CAF/actor-framework/build/libcaf_core'

# AKKA flags
sbtflags="--warn -J-Xmx32g"

# UCPP flags
UCPPflags="-quiet -g -Wall -Wextra -O3 -nodebug -DNDEBUG -multi"
UCPP=~/ucpp/u++-7.0.0/bin/u++

# run the benchmarks

# /usr/bin/time -f "%Uu %Ss %Er %Mkb"
if [ ${executor} -eq ${true} ] ; then
    echo "executor"
    pre_args="40000 100 ${nRounds}"
    post_args="${batch}"
    cd cfa # CFA RUN
    for i in ${!names[@]}; do
        print_header ${names[$i]}
        ${cfa} ${cfa_flags} ${var_flags[$i]} executor.cfa -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
    done
    cd - > /dev/null # done CFA
    if [ ${runCAF} -eq ${true} ] ; then # CAF RUN
        cd caf
        print_header 'CAF'
        g++ ${prefixCAF} CAFExecutor.cpp ${suffixCAF} -o a.${hostname} > /dev/null 2>&1
        for p in ${bench_cores} ; do
            sed -i "s/max-threads = .*/max-threads = ${p}/g" caf-application.conf # set CAF parallelism
            single_run ${p}
        done
        rm a.${hostname}
        # set back parallelism
        sed -i "s/max-threads = .*/max-threads = 1/g" caf-application.conf 
        cd - > /dev/null
    fi # done CAF
    if [ ${runAKKA} -eq ${true} ] ; then # AKKA RUN
        cd akka/Executor
        rm -rf project target				# random out of memory errors without this
        print_header 'Akka'
        for p in ${bench_cores} ; do
            set_akka_parallelism ${p}
            affinity ${p}
            taskset -c ${taskset} sbt ${sbtflags} "run ${pre_args} ${p} ${post_args} ${numtimes}" 2>&1 | grep -v "SLF4J:" | grep -v "Slf4jLogger started" | sed -e "s/\x1b\[0J//"
            sbt clean > /dev/null
        done
        # set back parallelism
        set_akka_parallelism 1
        cd - > /dev/null
    fi # done AKKA
    if [ ${runUCPP} -eq ${true} ] ; then # UCPP RUN
        cd ucpp
        print_header 'uC++'
        ${UCPP} ${UCPPflags} uC++Executor.cc -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
        cd - > /dev/null
    fi # done UCPP
    if [ ${runPROTO} -eq ${true} ] ; then # PROTO RUN
        print_header 'ProtoActor'
        cd proto/Executor
        go build -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
        cd - > /dev/null
    fi # done PROTO
    echo ""
fi

if [ ${matrix} -eq ${true} ] ; then
    echo "matrix"
    pre_args="${size} ${size} ${size}"
    post_args=""
    cd cfa
    for i in ${!names[@]}; do
        print_header ${names[$i]}
        ${cfa} ${cfa_flags} ${var_flags[$i]} matrix.cfa -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
    done
    cd - > /dev/null
    if [ ${runCAF} -eq ${true} ] ; then # CAF RUN
        cd caf
        print_header 'CAF'
        g++ ${prefixCAF} CAFMatrix.cpp ${suffixCAF} -o a.${hostname} > /dev/null 2>&1
        for p in ${bench_cores} ; do
            sed -i "s/max-threads = .*/max-threads = ${p}/g" caf-application.conf # set CAF parallelism
            single_run ${p}
        done
        rm a.${hostname}

        # set back parallelism
        sed -i "s/max-threads = .*/max-threads = 1/g" caf-application.conf 
        cd - > /dev/null
    fi # done CAF
    if [ ${runAKKA} -eq ${true} ] ; then # AKKA RUN
        cd akka/Matrix
        rm -rf project target				# random out of memory errors without this
        print_header 'Akka'
        for p in ${bench_cores} ; do
            set_akka_parallelism ${p}
            affinity ${p}
            repeat_command taskset -c ${taskset} sbt ${sbtflags} "run ${pre_args} ${p}" 2>&1 | grep -v "SLF4J:" | grep -v "Slf4jLogger started" | sed -e "s/\x1b\[0J//"
            sbt clean > /dev/null
        done
        # set back parallelism
        set_akka_parallelism 1
        cd - > /dev/null
    fi # done AKKA
    if [ ${runUCPP} -eq ${true} ] ; then # UCPP RUN
        cd ucpp
        print_header 'uC++'
        ${UCPP} ${UCPPflags} uC++Matrix.cc -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
        cd - > /dev/null
    fi # done UCPP
    if [ ${runPROTO} -eq ${true} ] ; then # PROTO RUN
        cd proto/Matrix
        print_header 'ProtoActor'
        go build -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
        cd - > /dev/null
    fi # done PROTO
    echo ""
fi

if [ ${repeat} -eq ${true} ] ; then
    echo "repeat"
    pre_args="${messages}"
    post_args="${n_repeats}"
    cd cfa
    for i in ${!names[@]}; do
        print_header ${names[$i]}
        ${cfa} ${cfa_flags} ${var_flags[$i]} repeat.cfa -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
    done
    cd - > /dev/null
    if [ ${runCAF} -eq ${true} ] ; then # CAF RUN
        cd caf
        print_header 'CAF'
        g++ ${prefixCAF} CAFRepeat.cpp ${suffixCAF} -o a.${hostname} > /dev/null 2>&1
        for p in ${bench_cores} ; do
            sed -i "s/max-threads = .*/max-threads = ${p}/g" caf-application.conf # set CAF parallelism
            single_run ${p}
        done
        rm a.${hostname}

        # set back parallelism
        sed -i "s/max-threads = .*/max-threads = 1/g" caf-application.conf 
        cd - > /dev/null
    fi # done CAF
    if [ ${runAKKA} -eq ${true} ] ; then # AKKA RUN
        cd akka/Repeat
        rm -rf project target				# random out of memory errors without this
        print_header 'Akka'
        for p in ${bench_cores} ; do
            set_akka_parallelism ${p}
            affinity ${p}
            repeat_command taskset -c ${taskset} sbt ${sbtflags} "run ${pre_args} ${p} ${post_args}" 2>&1 | grep -v "SLF4J:" | grep -v "Slf4jLogger started" | sed -e "s/\x1b\[0J//"
            sbt clean > /dev/null
        done
        # set back parallelism
        set_akka_parallelism 1
        cd - > /dev/null
    fi # done AKKA
    if [ ${runUCPP} -eq ${true} ] ; then # UCPP RUN
        cd ucpp
        print_header 'uC++'
        ${UCPP} ${UCPPflags} uC++Repeat.cc -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
        cd - > /dev/null
    fi # done UCPP
    if [ ${runPROTO} -eq ${true} ] ; then # PROTO RUN
        print_header 'ProtoActor'
        cd proto/Repeat
        go build -o a.${hostname} > /dev/null 2>&1
        run_bench
        rm a.${hostname}
        cd - > /dev/null
    fi # done PROTO
    echo ""
fi


if [ ${static} -eq ${true} ] ; then
    echo "static"
    preprint=''
    cd cfa
    for i in ${!names[@]}; do
        echo ${names[$i]}
        ${cfa} ${cfa_flags} ${var_flags[$i]} static.cfa -o a.${hostname} > /dev/null 2>&1
        repeat_command taskset -c ${startcore} ./a.${hostname} ${n_static_sends}
        rm a.${hostname}
    done
    cd - > /dev/null
    if [ ${runCAF} -eq ${true} ] ; then # CAF RUN
        cd caf
        echo 'CAF:'
        g++ ${prefixCAF} CAFSendStatic.cpp ${suffixCAF} -o a.${hostname} > /dev/null 2>&1
        sed -i "s/max-threads = .*/max-threads = 1/g" caf-application.conf # set CAF parallelism
        repeat_command taskset -c ${startcore} ./a.${hostname} 'd'
        rm a.${hostname}
        cd - > /dev/null
    fi # done CAF
    if [ ${runAKKA} -eq ${true} ] ; then # AKKA RUN
        cd akka/SendStatic
        rm -rf project target				# random out of memory errors without this
        echo 'Akka:'
        set_akka_parallelism 1
        taskset -c ${startcore} sbt ${sbtflags} "run ${n_static_sends} ${numtimes}" 2>&1 | grep -v "SLF4J:" | grep -v "Slf4jLogger started" | sed -e "s/\x1b\[0J//"
        cd - > /dev/null
    fi # done AKKA
    if [ ${runUCPP} -eq ${true} ] ; then # UCPP RUN
        cd ucpp
        echo 'uC++:'
        ${UCPP} ${UCPPflags} uC++SendStatic.cc -o a.${hostname} > /dev/null 2>&1
        repeat_command taskset -c ${startcore} ./a.${hostname} ${n_static_sends}
        rm a.${hostname}
        cd - > /dev/null
    fi # done UCPP
    if [ ${runPROTO} -eq ${true} ] ; then # PROTO RUN
        cd proto/SendStatic
        echo 'ProtoActor:'
        go build -o a.${hostname} > /dev/null 2>&1
        repeat_command taskset -c ${startcore} ./a.${hostname} ${n_static_sends}
        rm a.${hostname}
        cd - > /dev/null
    fi # done PROTO
    echo ""
fi

if [ ${dynamic} -eq ${true} ] ; then
    echo "dynamic"
    cd cfa
    for i in ${!names[@]}; do
        echo ${names[$i]}
        ${cfa} ${cfa_flags} ${var_flags[$i]} dynamic.cfa -o a.${hostname} > /dev/null 2>&1
        repeat_command taskset -c ${startcore} ./a.${hostname} ${n_dynamic_sends}
        rm a.${hostname}
    done
    cd - > /dev/null
    if [ ${runCAF} -eq ${true} ] ; then # CAF RUN
        cd caf
        echo 'CAF:'
        g++ ${prefixCAF} CAFSendDynamic.cpp ${suffixCAF} -o a.${hostname} > /dev/null 2>&1
        sed -i "s/max-threads = .*/max-threads = 1/g" caf-application.conf # set CAF parallelism
        repeat_command taskset -c ${startcore} ./a.${hostname} 'd'
        rm a.${hostname}
        cd - > /dev/null
    fi # done CAF
    if [ ${runAKKA} -eq ${true} ] ; then # AKKA RUN
        cd akka/SendDynamic
        rm -rf project target				# random out of memory errors without this
        echo 'Akka:'
        set_akka_parallelism 1
        taskset -c ${startcore} sbt ${sbtflags} "run d ${numtimes}" 2>&1 | grep -v "SLF4J:" | grep -v "Slf4jLogger started" | sed -e "s/\x1b\[0J//"
        cd - > /dev/null
    fi # done AKKA
    if [ ${runUCPP} -eq ${true} ] ; then # UCPP RUN
        cd ucpp
        echo 'uC++:'
        ${UCPP} ${UCPPflags} uC++SendDynamic.cc -o a.${hostname} > /dev/null 2>&1
        repeat_command taskset -c ${startcore} ./a.${hostname} ${n_dynamic_sends}
        rm a.${hostname}
        cd - > /dev/null
    fi # done UCPP
    if [ ${runPROTO} -eq ${true} ] ; then # PROTO RUN
        cd proto/SendDynamic
        echo 'ProtoActor:'
        go build -o a.${hostname} > /dev/null 2>&1
        repeat_command taskset -c ${startcore} ./a.${hostname} 'd'
        rm a.${hostname}
        cd - > /dev/null
    fi # done PROTO
    echo ""
fi


if [ ${balance} -eq ${true} ] ; then
    cd cfa
    echo "balance_one"
    for i in ${!names[@]}; do
        echo ${names[$i]}':'
        echo -e "proc\ttime (s)"
        ${cfa} ${cfa_flags} ${var_flags[$i]} balance.cfa -o a.${hostname} > /dev/null 2>&1
        for p in ${bench_cores} ; do 
            affinity ${p}
            preprint="${p}\t"
            repeat_command taskset -c ${taskset} ./a.${hostname} 32 32 ${nOneRounds} ${p}
        done
        rm a.${hostname}
    done
    echo ""
    echo "balance_multi"
    for i in ${!names[@]}; do
        echo ${names[$i]}':'
        echo -e "proc\ttime (s)"
        ${cfa} ${cfa_flags} ${var_flags[$i]} -DMULTI balance.cfa -o a.${hostname} > /dev/null 2>&1
        for p in ${bench_cores} ; do 
            affinity ${p}
            preprint="${p}\t"
            repeat_command taskset -c ${taskset} ./a.${hostname} 32 32 ${nMultiRounds} ${p}
        done
        rm a.${hostname}
    done
    echo ""
    cd - > /dev/null
fi