source: benchmark/io/http/filecache.cfa @ cf191ac

#include "filecache.hfa"

#include <assert.h>
#include <string.h>

#include <fstream.hfa>
#include <stdlib.hfa>

#include <errno.h>
#include <unistd.h>
extern "C" {
        #include <fcntl.h>
        #include <ftw.h>
}

#include "options.hfa"

static inline uint32_t murmur_32_scramble(uint32_t k) {
    k *= 0xcc9e2d51;
    k = (k << 15) | (k >> 17);
    k *= 0x1b873593;
    return k;
}

uint32_t murmur3_32(const uint8_t* key, size_t len, uint32_t seed)
{
        uint32_t h = seed;
        uint32_t k;
        /* Read in groups of 4. */
        for (size_t i = len >> 2; i; i--) {
                // Here is a source of differing results across endiannesses.
                // A swap here has no effects on hash properties though.
                memcpy(&k, key, sizeof(uint32_t));
                key += sizeof(uint32_t);
                h ^= murmur_32_scramble(k);
                h = (h << 13) | (h >> 19);
                h = h * 5 + 0xe6546b64;
        }
        /* Read the rest. */
        k = 0;
        for (size_t i = len & 3; i; i--) {
                k <<= 8;
                k |= key[i - 1];
        }
        // A swap is *not* necessary here because the preceding loop already
        // places the low bytes in the low places according to whatever endianness
        // we use. Swaps only apply when the memory is copied in a chunk.
        h ^= murmur_32_scramble(k);
        /* Finalize. */
        h ^= len;
        h ^= h >> 16;
        h *= 0x85ebca6b;
        h ^= h >> 13;
        h *= 0xc2b2ae35;
        h ^= h >> 16;
        return h;
}

static inline [unsigned size, char unit] human_size( size_t size ) {
        int idx = 0;
        static char units [] = { ' ', 'K', 'M', 'G', 'T' };
        while( size >= 1024 ) {
                idx++;
                size /= 1024;
                if(idx >= 5) {
                        abort("File too large to print\n");
                }
        }

        return [size, units[idx]];
}

struct {
        cache_line * entries;
        size_t size;
        int * rawfds;
        int nfds;
} file_cache;

void ?{}( cache_line & this ) with( this ) {
        file = 0p;
        size = 0;
        fd   = 0;
}

[int fd, size_t size] get_file( * const char file, size_t len ) {
        uint32_t idx = murmur3_32( (const uint8_t *)file, len, options.file_cache.hash_seed ) % file_cache.size;

        for(int i = 0;; i++) {
                assert( i < file_cache.size );
                cache_line & entry = file_cache.entries[idx];
                if( !entry.file ) return [-1, 0];
                #if !defined(REJECT_CONFLICTS)
                        if( strncmp(entry.file, file, len) != 0 ) {
                                idx = (idx + 1) % file_cache.size;
                                continue;
                        }
                #endif
                return [entry.fd, entry.size];
        }
}

int put_file( cache_line & entry, int fd ) {
        uint32_t idx = murmur3_32( (const uint8_t *)entry.file, strlen(entry.file), options.file_cache.hash_seed ) % file_cache.size;

        int i = 0;
        for(;file_cache.entries[idx].file; i++ ) {
                assert( i < file_cache.size );
                idx = (idx + 1) % file_cache.size;
        }

        file_cache.entries[idx] = entry;
        file_cache.entries[idx].fd = fd;
        return i > 0 ? 1 : 0;
}

// int ftw(const char *dirpath, int (*fn) (const char *fpath, const struct stat *sb, int typeflag), int nopenfd)
void fill_cache( const char * path ) {
        int ret;
        ret = chdir(path);
        if(ret < 0) {
                abort( "chdir error: (%d) %s\n", (int)errno, strerror(errno) );
        }

        size_t fcount = 0;
        size_t fsize = 16;
        cache_line * raw = alloc(fsize);
        // Step 1 get a dense array of all files
        int walk(const char *fpath, const struct stat *sb, int typeflag) {
                if(typeflag != FTW_F) return 0;

                int idx = fcount;
                fcount++;
                if(fcount > fsize) {
                        fsize *= 2;
                        raw = alloc(fsize, raw`realloc);
                }

                raw[idx].file = strdup(fpath+2);
                raw[idx].size = sb->st_size;
                if( !options.file_cache.list ) {
                        raw[idx].fd = open( fpath, options.file_cache.open_flags );
                        if(raw[idx].fd < 0) {
                                abort( "open file error: (%d) %s\n", (int)errno, strerror(errno) );
                        }
                }
                return 0;
        }

        ret = ftw(".", walk, 10);
        if(ret < 0) {
                abort( "ftw error: (%d) %s\n", (int)errno, strerror(errno) );
        }

        if(fcount == 0) {
                abort("No file found in path %s\n", path);
        }

        // Step 2 create the cache
        file_cache.size = options.file_cache.size > 0 ? options.file_cache.size : fsize;
        if( file_cache.size < fcount ) {
                abort("File Cache too small\n");
        }

        file_cache.entries = anew(file_cache.size);

        if(options.file_cache.fixed_fds) {
                file_cache.nfds   = fcount;
                file_cache.rawfds = alloc(fcount);
        }

        // Step 3 fill the cache
        int conflicts = 0;
        for(i; fcount) {
                int fd;
                if(options.file_cache.fixed_fds) {
                        file_cache.rawfds[i] = raw[i].fd;
                        fd = i;
                }
                else {
                        fd = raw[i].fd;
                }
                conflicts += put_file( raw[i], fd );
        }
        sout | "Filled cache from path \"" | path | "\" with" | fcount | "files";
        if( conflicts > 0 ) {
                sout | "Found" | conflicts | "conflicts (size: " | file_cache.size | ", seed: " | options.file_cache.hash_seed | ")";
                #if defined(REJECT_CONFLICTS)
                        abort("Conflicts found in the cache");
                #endif
        }

        if(options.file_cache.list) {
                sout | "Listing files and exiting";
                for(i; fcount) {
                        int s; char u;
                        [s, u] = human_size(raw[i].size);
                        sout | s | u | "-" | raw[i].file;
                        free(raw[i].file);
                }
                free(raw);
                adelete( file_cache.entries );
                exit(0);
        }

        // Step 4 clean up
        free( raw );
}

[int *, int] filefds(int extra) {
        if(!options.file_cache.path) {
                int * data = alloc(extra);
                return [data, 0];
        }

        if(!file_cache.entries) {
                abort("File cache not filled!\n");
        }

        size_t s = file_cache.nfds + extra;
        int * data = alloc(s, file_cache.rawfds`realloc);
        return [data, file_cache.nfds];
}


void close_cache() {
        for(idx; file_cache.size) {
                cache_line & entry = file_cache.entries[idx];
                if( !entry.file ) continue;

                int ret = close( entry.fd );
                if(ret < 0) {
                        abort( "close file error: (%d) %s\n", (int)errno, strerror(errno) );
                }
        }

        delete( file_cache.entries );
}