#include "filecache.hfa" #include #include #include #include #include extern "C" { #include #include } #include "options.hfa" static inline uint32_t murmur_32_scramble(uint32_t k) { k *= 0xcc9e2d51; k = (k << 15) | (k >> 17); k *= 0x1b873593; return k; } uint32_t murmur3_32(const uint8_t* key, size_t len, uint32_t seed) { uint32_t h = seed; uint32_t k; /* Read in groups of 4. */ for (size_t i = len >> 2; i; i--) { // Here is a source of differing results across endiannesses. // A swap here has no effects on hash properties though. memcpy(&k, key, sizeof(uint32_t)); key += sizeof(uint32_t); h ^= murmur_32_scramble(k); h = (h << 13) | (h >> 19); h = h * 5 + 0xe6546b64; } /* Read the rest. */ k = 0; for (size_t i = len & 3; i; i--) { k <<= 8; k |= key[i - 1]; } // A swap is *not* necessary here because the preceding loop already // places the low bytes in the low places according to whatever endianness // we use. Swaps only apply when the memory is copied in a chunk. h ^= murmur_32_scramble(k); /* Finalize. */ h ^= len; h ^= h >> 16; h *= 0x85ebca6b; h ^= h >> 13; h *= 0xc2b2ae35; h ^= h >> 16; return h; } struct { cache_line * entries; size_t size; } file_cache; void ?{}( cache_line & this ) with( this ) { file = 0p; size = 0; fd = 0; } [int fd, size_t size] get_file( * const char file, size_t len ) { uint32_t idx = murmur3_32( (const uint8_t *)file, len, options.hash_seed ) % file_cache.size; for(int i = 0;; i++) { assert( i < file_cache.size ); cache_line & entry = file_cache.entries[idx]; if( !entry.file ) return [-1, 0]; #if !defined(REJECT_CONFLICTS) if( strncmp(entry.file, file, len) != 0 ) { idx = (idx + 1) % file_cache.size; continue; } #endif return [entry.fd, entry.size]; } } int put_file( cache_line & entry ) { uint32_t idx = murmur3_32( (const uint8_t *)entry.file, strlen(entry.file), options.hash_seed ) % file_cache.size; int i = 0; for(;file_cache.entries[idx].file; i++ ) { assert( i < file_cache.size ); idx = (idx + 1) % file_cache.size; } file_cache.entries[idx] = entry; return i > 0 ? 1 : 0; } // int ftw(const char *dirpath, int (*fn) (const char *fpath, const struct stat *sb, int typeflag), int nopenfd) void fill_cache( const char * path ) { int ret; size_t fcount = 0; size_t fsize = 16; cache_line * raw = 0p; raw = alloc(raw, fsize, true); // Step 1 get a dense array of all files int walk(const char *fpath, const struct stat *sb, int typeflag) { if(typeflag != FTW_F) return 0; int idx = fcount; fcount++; if(fcount > fsize) { fsize *= 2; raw = alloc(raw, fsize, true); } raw[idx].file = strdup(fpath+2); raw[idx].size = sb->st_size; raw[idx].fd = open( fpath, options.open_flags ); if(raw[idx].fd < 0) { abort( "open file error: (%d) %s\n", (int)errno, strerror(errno) ); } return 0; } ret = ftw(path, walk, 10); if(ret < 0) { abort( "ftw error: (%d) %s\n", (int)errno, strerror(errno) ); } if(fcount == 0) { abort("No file found in path %s\n", path); } // Step 2 create the cache file_cache.size = options.file_cache_size > 0 ? options.file_cache_size : fsize; if( file_cache.size < fcount ) { abort("File Cache too small\n"); } file_cache.entries = anew(fsize); // Step 3 fill the cache int conflicts = 0; for(i; fcount) { printf("Added file %s\n", raw[i].file); conflicts += put_file( raw[i] ); } printf("Filled cache from path \"%s\" with %zu files\n", path, fcount); if( conflicts > 0 ) { printf("Found %d conflicts (seed: %u)\n", conflicts, options.hash_seed); #if defined(REJECT_CONFLICTS) abort("Conflicts found in the cache"); #endif } // Step 4 clean up free( raw ); } void close_cache() { for(idx; file_cache.size) { cache_line & entry = file_cache.entries[idx]; if( !entry.file ) continue; int ret = close( entry.fd ); if(ret < 0) { abort( "close file error: (%d) %s\n", (int)errno, strerror(errno) ); } } delete( file_cache.entries ); }