#include #include #include // STDERR_FILENO #include #include "bits/debug.hfa" #include "bits/locks.hfa" #include "stats.hfa" #if !defined(__CFA_NO_STATISTICS__) void __init_stats( struct __stats_t * stats ) { stats->ready.push.local.attempt = 0; stats->ready.push.local.success = 0; stats->ready.push.share.attempt = 0; stats->ready.push.share.success = 0; stats->ready.push.extrn.attempt = 0; stats->ready.push.extrn.success = 0; stats->ready.pop.local .attempt = 0; stats->ready.pop.local .success = 0; stats->ready.pop.local .elock = 0; stats->ready.pop.local .eempty = 0; stats->ready.pop.local .espec = 0; stats->ready.pop.help .attempt = 0; stats->ready.pop.help .success = 0; stats->ready.pop.help .elock = 0; stats->ready.pop.help .eempty = 0; stats->ready.pop.help .espec = 0; stats->ready.pop.steal .attempt = 0; stats->ready.pop.steal .success = 0; stats->ready.pop.steal .elock = 0; stats->ready.pop.steal .eempty = 0; stats->ready.pop.steal .espec = 0; stats->ready.pop.search.attempt = 0; stats->ready.pop.search.success = 0; stats->ready.pop.search.elock = 0; stats->ready.pop.search.eempty = 0; stats->ready.pop.search.espec = 0; stats->ready.threads.migration = 0; stats->ready.threads.threads = 0; stats->ready.sleep.halts = 0; stats->ready.sleep.cancels = 0; stats->ready.sleep.wakes = 0; stats->ready.sleep.exits = 0; #if defined(CFA_HAVE_LINUX_IO_URING_H) stats->io.alloc.fast = 0; stats->io.alloc.slow = 0; stats->io.alloc.fail = 0; stats->io.alloc.revoke = 0; stats->io.alloc.block = 0; stats->io.submit.fast = 0; stats->io.submit.slow = 0; stats->io.flush.external = 0; stats->io.calls.flush = 0; stats->io.calls.submitted = 0; stats->io.calls.drain = 0; stats->io.calls.completed = 0; stats->io.calls.errors.busy = 0; stats->io.poller.sleeps = 0; #endif #if defined(CFA_STATS_ARRAY) stats->array.values = alloc(CFA_STATS_ARRAY); stats->array.cnt = 0; #endif } void __tally_stats( struct __stats_t * cltr, struct __stats_t * proc ) { __atomic_fetch_add( &cltr->ready.push.local.attempt, proc->ready.push.local.attempt, __ATOMIC_SEQ_CST ); proc->ready.push.local.attempt = 0; __atomic_fetch_add( &cltr->ready.push.local.success, proc->ready.push.local.success, __ATOMIC_SEQ_CST ); proc->ready.push.local.success = 0; __atomic_fetch_add( &cltr->ready.push.share.attempt, proc->ready.push.share.attempt, __ATOMIC_SEQ_CST ); proc->ready.push.share.attempt = 0; __atomic_fetch_add( &cltr->ready.push.share.success, proc->ready.push.share.success, __ATOMIC_SEQ_CST ); proc->ready.push.share.success = 0; __atomic_fetch_add( &cltr->ready.push.extrn.attempt, proc->ready.push.extrn.attempt, __ATOMIC_SEQ_CST ); proc->ready.push.extrn.attempt = 0; __atomic_fetch_add( &cltr->ready.push.extrn.success, proc->ready.push.extrn.success, __ATOMIC_SEQ_CST ); proc->ready.push.extrn.success = 0; __atomic_fetch_add( &cltr->ready.pop.local .attempt, proc->ready.pop.local .attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.local .attempt = 0; __atomic_fetch_add( &cltr->ready.pop.local .success, proc->ready.pop.local .success, __ATOMIC_SEQ_CST ); proc->ready.pop.local .success = 0; __atomic_fetch_add( &cltr->ready.pop.local .elock , proc->ready.pop.local .elock , __ATOMIC_SEQ_CST ); proc->ready.pop.local .elock = 0; __atomic_fetch_add( &cltr->ready.pop.local .eempty , proc->ready.pop.local .eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.local .eempty = 0; __atomic_fetch_add( &cltr->ready.pop.local .espec , proc->ready.pop.local .espec , __ATOMIC_SEQ_CST ); proc->ready.pop.local .espec = 0; __atomic_fetch_add( &cltr->ready.pop.help .attempt, proc->ready.pop.help .attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.help .attempt = 0; __atomic_fetch_add( &cltr->ready.pop.help .success, proc->ready.pop.help .success, __ATOMIC_SEQ_CST ); proc->ready.pop.help .success = 0; __atomic_fetch_add( &cltr->ready.pop.help .elock , proc->ready.pop.help .elock , __ATOMIC_SEQ_CST ); proc->ready.pop.help .elock = 0; __atomic_fetch_add( &cltr->ready.pop.help .eempty , proc->ready.pop.help .eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.help .eempty = 0; __atomic_fetch_add( &cltr->ready.pop.help .espec , proc->ready.pop.help .espec , __ATOMIC_SEQ_CST ); proc->ready.pop.help .espec = 0; __atomic_fetch_add( &cltr->ready.pop.steal .attempt, proc->ready.pop.steal .attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.steal .attempt = 0; __atomic_fetch_add( &cltr->ready.pop.steal .success, proc->ready.pop.steal .success, __ATOMIC_SEQ_CST ); proc->ready.pop.steal .success = 0; __atomic_fetch_add( &cltr->ready.pop.steal .elock , proc->ready.pop.steal .elock , __ATOMIC_SEQ_CST ); proc->ready.pop.steal .elock = 0; __atomic_fetch_add( &cltr->ready.pop.steal .eempty , proc->ready.pop.steal .eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.steal .eempty = 0; __atomic_fetch_add( &cltr->ready.pop.steal .espec , proc->ready.pop.steal .espec , __ATOMIC_SEQ_CST ); proc->ready.pop.steal .espec = 0; __atomic_fetch_add( &cltr->ready.pop.search.attempt, proc->ready.pop.search.attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.search.attempt = 0; __atomic_fetch_add( &cltr->ready.pop.search.success, proc->ready.pop.search.success, __ATOMIC_SEQ_CST ); proc->ready.pop.search.success = 0; __atomic_fetch_add( &cltr->ready.pop.search.elock , proc->ready.pop.search.elock , __ATOMIC_SEQ_CST ); proc->ready.pop.search.elock = 0; __atomic_fetch_add( &cltr->ready.pop.search.eempty , proc->ready.pop.search.eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.search.eempty = 0; __atomic_fetch_add( &cltr->ready.pop.search.espec , proc->ready.pop.search.espec , __ATOMIC_SEQ_CST ); proc->ready.pop.search.espec = 0; __atomic_fetch_add( &cltr->ready.threads.migration , proc->ready.threads.migration , __ATOMIC_SEQ_CST ); proc->ready.threads.migration = 0; __atomic_fetch_add( &cltr->ready.threads.threads , proc->ready.threads.threads , __ATOMIC_SEQ_CST ); proc->ready.threads.threads = 0; __atomic_fetch_add( &cltr->ready.sleep.halts , proc->ready.sleep.halts , __ATOMIC_SEQ_CST ); proc->ready.sleep.halts = 0; __atomic_fetch_add( &cltr->ready.sleep.cancels , proc->ready.sleep.cancels , __ATOMIC_SEQ_CST ); proc->ready.sleep.cancels = 0; __atomic_fetch_add( &cltr->ready.sleep.wakes , proc->ready.sleep.wakes , __ATOMIC_SEQ_CST ); proc->ready.sleep.wakes = 0; __atomic_fetch_add( &cltr->ready.sleep.exits , proc->ready.sleep.exits , __ATOMIC_SEQ_CST ); proc->ready.sleep.exits = 0; #if defined(CFA_HAVE_LINUX_IO_URING_H) __atomic_fetch_add( &cltr->io.alloc.fast , proc->io.alloc.fast , __ATOMIC_SEQ_CST ); proc->io.alloc.fast = 0; __atomic_fetch_add( &cltr->io.alloc.slow , proc->io.alloc.slow , __ATOMIC_SEQ_CST ); proc->io.alloc.slow = 0; __atomic_fetch_add( &cltr->io.alloc.fail , proc->io.alloc.fail , __ATOMIC_SEQ_CST ); proc->io.alloc.fail = 0; __atomic_fetch_add( &cltr->io.alloc.revoke , proc->io.alloc.revoke , __ATOMIC_SEQ_CST ); proc->io.alloc.revoke = 0; __atomic_fetch_add( &cltr->io.alloc.block , proc->io.alloc.block , __ATOMIC_SEQ_CST ); proc->io.alloc.block = 0; __atomic_fetch_add( &cltr->io.submit.fast , proc->io.submit.fast , __ATOMIC_SEQ_CST ); proc->io.submit.fast = 0; __atomic_fetch_add( &cltr->io.submit.slow , proc->io.submit.slow , __ATOMIC_SEQ_CST ); proc->io.submit.slow = 0; __atomic_fetch_add( &cltr->io.flush.external , proc->io.flush.external , __ATOMIC_SEQ_CST ); proc->io.flush.external = 0; __atomic_fetch_add( &cltr->io.calls.flush , proc->io.calls.flush , __ATOMIC_SEQ_CST ); proc->io.calls.flush = 0; __atomic_fetch_add( &cltr->io.calls.submitted , proc->io.calls.submitted , __ATOMIC_SEQ_CST ); proc->io.calls.submitted = 0; __atomic_fetch_add( &cltr->io.calls.drain , proc->io.calls.drain , __ATOMIC_SEQ_CST ); proc->io.calls.drain = 0; __atomic_fetch_add( &cltr->io.calls.completed , proc->io.calls.completed , __ATOMIC_SEQ_CST ); proc->io.calls.completed = 0; __atomic_fetch_add( &cltr->io.calls.errors.busy, proc->io.calls.errors.busy, __ATOMIC_SEQ_CST ); proc->io.calls.errors.busy = 0; __atomic_fetch_add( &cltr->io.poller.sleeps , proc->io.poller.sleeps , __ATOMIC_SEQ_CST ); proc->io.poller.sleeps = 0; #endif } void __print_stats( struct __stats_t * stats, int flags, const char * type, const char * name, void * id ) with( *stats ) { if( flags & CFA_STATS_READY_Q ) { double push_len = ((double)ready.push.local.attempt + ready.push.share.attempt + ready.push.extrn.attempt) / (ready.push.local.success + ready.push.share.success + ready.push.extrn.success); double sLcl_len = ready.push.local.success ? ((double)ready.push.local.attempt) / ready.push.local.success : 0; double sOth_len = ready.push.share.success ? ((double)ready.push.share.attempt) / ready.push.share.success : 0; double sExt_len = ready.push.extrn.success ? ((double)ready.push.extrn.attempt) / ready.push.extrn.success : 0; uint64_t total = ready.pop.local.success + ready.pop.help.success + ready.pop.steal.success + ready.pop.search.success; double rLcl_pc = (100.0 * (double)ready.pop.local .success) / total; double rHlp_pc = (100.0 * (double)ready.pop.help .success) / total; double rStl_pc = (100.0 * (double)ready.pop.steal .success) / total; double rSch_pc = (100.0 * (double)ready.pop.search.success) / total; __cfaabi_bits_print_safe( STDOUT_FILENO, "----- %s \"%s\" (%p) - Ready Q Stats -----\n" "- totals : %'3" PRIu64 " run, %'3" PRIu64 " schd (%'" PRIu64 "ext, %'" PRIu64 "mig, %'" PRId64 " )\n" "- push avg : %'3.0lf (l: %'3.2lf/%'" PRIu64 ", s: %'3.2lf/%'" PRIu64 ", e: %'3.2lf : %'" PRIu64 "e)\n" "- local : %'3.0lf%%: %'3" PRIu64 " (%'3" PRIu64 " try, %'3" PRIu64 " spc, %'3" PRIu64 " lck, %'3" PRIu64 " ept)\n" "- help : %'3.0lf%%: %'3" PRIu64 " (%'3" PRIu64 " try, %'3" PRIu64 " spc, %'3" PRIu64 " lck, %'3" PRIu64 " ept)\n" "- steal : %'3.0lf%%: %'3" PRIu64 " (%'3" PRIu64 " try, %'3" PRIu64 " spc, %'3" PRIu64 " lck, %'3" PRIu64 " ept)\n" "- search : %'3.0lf%%: %'3" PRIu64 " (%'3" PRIu64 " try, %'3" PRIu64 " spc, %'3" PRIu64 " lck, %'3" PRIu64 " ept)\n" "- Idle Slp : %'3" PRIu64 "h, %'3" PRIu64 "c, %'3" PRIu64 "w, %'3" PRIu64 "e\n" "\n" , type, name, id , total , ready.push.local.success + ready.push.share.success + ready.push.extrn.success , ready.push.extrn.success, ready.threads.migration, ready.threads.threads , push_len, sLcl_len, ready.push.local.attempt, sOth_len, ready.push.share.attempt, sExt_len, ready.push.extrn.attempt , rLcl_pc, ready.pop.local .success, ready.pop.local .attempt, ready.pop.local .espec, ready.pop.local .elock, ready.pop.local .eempty , rHlp_pc, ready.pop.help .success, ready.pop.help .attempt, ready.pop.help .espec, ready.pop.help .elock, ready.pop.help .eempty , rStl_pc, ready.pop.steal .success, ready.pop.steal .attempt, ready.pop.steal .espec, ready.pop.steal .elock, ready.pop.steal .eempty , rSch_pc, ready.pop.search.success, ready.pop.search.attempt, ready.pop.search.espec, ready.pop.search.elock, ready.pop.search.eempty , ready.sleep.halts, ready.sleep.cancels, ready.sleep.wakes, ready.sleep.exits ); } #if defined(CFA_HAVE_LINUX_IO_URING_H) if( flags & CFA_STATS_IO ) { uint64_t total_allocs = io.alloc.fast + io.alloc.slow; double avgfasta = ((double)io.alloc.fast) / total_allocs; uint64_t total_submits = io.submit.fast + io.submit.slow; double avgfasts = ((double)io.submit.fast) / total_submits; double avgsubs = ((double)io.calls.submitted) / io.calls.flush; double avgcomp = ((double)io.calls.completed) / io.calls.drain; __cfaabi_bits_print_safe( STDOUT_FILENO, "----- %s \"%s\" (%p) - I/O Stats -----\n" "- total allocations : %'" PRIu64 "f, %'" PRIu64 "s (%'2.2lff) \n" "- failures : %'" PRIu64 "oom, %'" PRIu64 "rvk, %'" PRIu64 "blk\n" "- total submits : %'" PRIu64 "f, %'" PRIu64 "s (%'2.2lf) \n" "- flush external : %'" PRIu64 "\n" "- io_uring_enter : %'" PRIu64 " (%'" PRIu64 ", %'" PRIu64 " EBUSY)\n" "- submits : %'" PRIu64 " (%'.2lf) \n" "- completes : %'" PRIu64 " (%'.2lf) \n" "- poller sleeping : %'" PRIu64 "\n" "\n" , type, name, id , io.alloc.fast, io.alloc.slow, avgfasta , io.alloc.fail, io.alloc.revoke, io.alloc.block , io.submit.fast, io.submit.slow, avgfasts , io.flush.external , io.calls.flush, io.calls.drain, io.calls.errors.busy , io.calls.submitted, avgsubs , io.calls.completed, avgcomp , io.poller.sleeps ); } #endif } #if defined(CFA_STATS_ARRAY) extern "C" { #include #include #include #include } void __flush_stat( struct __stats_t * this, const char * name, void * handle) { int ret = mkdir(".cfadata", 0755); if(ret < 0 && errno != EEXIST) abort("Failed to create directory .cfadata: %d\n", errno); char filename[100]; snprintf(filename, 100, ".cfadata/%s%p.data", name, handle); int fd = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0644); if(fd < 0) abort("Failed to create file %s: %d\n", filename, errno); for(i; this->array.cnt) { char line[100]; size_t n = snprintf(line, 100, "%llu, %lld\n", this->array.values[i].ts, this->array.values[i].value); write(fd, line, n); } this->array.cnt = 0; close(fd); } static __spinlock_t stats_lock; void __push_stat( struct __stats_t * this, int64_t value, bool external, const char * name, void * handle ) { if(external) lock(stats_lock __cfaabi_dbg_ctx2); if( this->array.cnt >= CFA_STATS_ARRAY ) __flush_stat( this, name, handle ); size_t idx = this->array.cnt; this->array.cnt++; if(external) unlock(stats_lock); this->array.values[idx].ts = rdtscl(); this->array.values[idx].value = value; } #endif #endif