[96f002c1] | 1 | // |
---|
| 2 | // Cforall Version 1.0.0 Copyright (C) 2021 University of Waterloo |
---|
| 3 | // |
---|
| 4 | // The contents of this file are covered under the licence agreement in the |
---|
| 5 | // file "LICENCE" distributed with Cforall. |
---|
| 6 | // |
---|
| 7 | // topology.cfa -- read the data structure |
---|
| 8 | // |
---|
| 9 | // Author : Thierry Delisle |
---|
| 10 | // Created On : Thu Jun 10 16:13:07 2021 |
---|
| 11 | // Last Modified By : |
---|
| 12 | // Last Modified On : |
---|
| 13 | // Update Count : |
---|
| 14 | // |
---|
| 15 | |
---|
| 16 | #include "device/cpu.hfa" |
---|
| 17 | |
---|
| 18 | #include <math.hfa> |
---|
| 19 | #include <stdlib.hfa> |
---|
| 20 | |
---|
| 21 | #include <errno.h> |
---|
| 22 | #include <stdio.h> |
---|
| 23 | #include <string.h> |
---|
| 24 | #include <unistd.h> |
---|
| 25 | |
---|
| 26 | extern "C" { |
---|
| 27 | #include <dirent.h> |
---|
| 28 | #include <sys/types.h> |
---|
| 29 | #include <sys/stat.h> |
---|
| 30 | #include <fcntl.h> |
---|
| 31 | } |
---|
| 32 | |
---|
| 33 | // search a string for character 'character' but looking atmost at len |
---|
| 34 | // chars |
---|
| 35 | static const char * strnchr(const char * str, int character, size_t len) { |
---|
| 36 | return (const char *)memchr(str, character, strnlen(str, len)); |
---|
| 37 | } |
---|
| 38 | |
---|
| 39 | // Check if have string matches the want string |
---|
| 40 | // ignoring any characters that are longer than the want string |
---|
| 41 | static bool strmatch(const char * want, char * have) { |
---|
| 42 | size_t w = strlen(want); |
---|
| 43 | return strncmp(want, have, w) == 0; |
---|
| 44 | } |
---|
| 45 | |
---|
| 46 | typedef const char * idx_range_t; |
---|
| 47 | |
---|
| 48 | // read the value of a string and evaluate it |
---|
| 49 | // get the end pointer and make sure it is all evaluated |
---|
| 50 | static unsigned read_value(idx_range_t map, size_t len, const char ** end) { |
---|
| 51 | unsigned long val = strtoul(map, (char**)end, 10); |
---|
| 52 | /* paranoid */ __attribute__((unused)) size_t read = (*end - map); |
---|
| 53 | /* paranoid */ verifyf(read <= len, "String '%s' passed with inconsistent length %zu", map, len); |
---|
| 54 | /* paranoid */ verifyf(read == len, "String %.*s not entirely a number, %zu chars left", (int)len, map, len - read); |
---|
| 55 | return val; |
---|
| 56 | } |
---|
| 57 | |
---|
| 58 | // Evaluate the width of a comma seperated list of idx |
---|
| 59 | // for example 'A-B,C-D,E,F' has a width of '(B-A) + (D-C) + 1 + 1' |
---|
| 60 | // Also has an (non-optional) end ptr like strtoul and friends |
---|
| 61 | // |
---|
| 62 | // FIXME : the current implementation only supports 1 comma |
---|
| 63 | static unsigned read_width(idx_range_t map, size_t len, const char ** end) { |
---|
| 64 | // Do we have a comma |
---|
| 65 | const char * comma = strnchr(map, ',', len); |
---|
| 66 | if(comma != 0p) { |
---|
| 67 | // We do! recurse and sum the widths |
---|
| 68 | const char * _; |
---|
| 69 | size_t split = comma - map; |
---|
| 70 | unsigned lhs = read_width(map, split, &_); |
---|
| 71 | unsigned rhs = read_width(comma + 1, len - split - 1, end); |
---|
| 72 | return lhs + rhs; |
---|
| 73 | } |
---|
| 74 | |
---|
| 75 | // No commas, check for a range |
---|
| 76 | const char * dash = strnchr(map, '-', len); |
---|
| 77 | if(dash != 0p) { |
---|
| 78 | const char * _; |
---|
| 79 | size_t split = dash - map; |
---|
| 80 | unsigned lhs = read_value(map, split, &_); |
---|
| 81 | unsigned rhs = read_value(dash + 1, len - split - 1, end); |
---|
| 82 | return rhs - lhs + 1; |
---|
| 83 | } |
---|
| 84 | |
---|
| 85 | // No range, no comma, just a single value |
---|
| 86 | // It's width is 1 and we can consume everything |
---|
| 87 | /* paranoid */ verifyf( ({strtoul(map, (char**)end, 10); *end == (map + len); }), "Value in range '%.*s' not a number", (int)len, map); |
---|
| 88 | *end = map + len; |
---|
| 89 | return 1; |
---|
| 90 | } |
---|
| 91 | |
---|
| 92 | // go through a directory calling fn on each file |
---|
| 93 | static int iterate_dir( const char * path, void (*fn)(struct dirent * ent) ) { |
---|
| 94 | // open the directory |
---|
| 95 | DIR *dir = opendir(path); |
---|
| 96 | if(dir == 0p) { return ENOTDIR; } |
---|
| 97 | |
---|
| 98 | // call fn for each |
---|
| 99 | struct dirent * ent; |
---|
| 100 | while ((ent = readdir(dir)) != 0p) { |
---|
| 101 | fn( ent ); |
---|
| 102 | } |
---|
| 103 | |
---|
| 104 | // no longer need this |
---|
| 105 | closedir(dir); |
---|
| 106 | return 0; |
---|
| 107 | } |
---|
| 108 | |
---|
| 109 | // count the number of directories with the specified prefix |
---|
| 110 | // the directories counted have the form '[prefix]N' where prefix is the parameter |
---|
| 111 | // and N is an base 10 integer. |
---|
| 112 | static int count_prefix_dirs(const char * path, const char * prefix) { |
---|
| 113 | // read the directory and find the cpu count |
---|
| 114 | // and make sure everything is as expected |
---|
| 115 | int max = -1; |
---|
| 116 | int count = 0; |
---|
| 117 | void lambda(struct dirent * ent) { |
---|
| 118 | // were are looking for prefixX, where X is a number |
---|
| 119 | // check that it starts with 'cpu |
---|
| 120 | char * s = strstr(ent->d_name, prefix); |
---|
| 121 | if(s == 0p) { return; } |
---|
| 122 | if(s != ent->d_name) { return; } |
---|
| 123 | |
---|
| 124 | // check that the next part is a number |
---|
| 125 | s += strlen(prefix); |
---|
| 126 | char * end; |
---|
| 127 | long int val = strtol(s, &end, 10); |
---|
| 128 | if(*end != '\0' || val < 0) { return; } |
---|
| 129 | |
---|
| 130 | // check that it's a directory |
---|
| 131 | if(ent->d_type != DT_DIR) { return; } |
---|
| 132 | |
---|
| 133 | // it's a match! |
---|
| 134 | max = max(val, max); |
---|
| 135 | count++; |
---|
| 136 | } |
---|
[8157bde] | 137 | int ret = iterate_dir(path, lambda); |
---|
| 138 | if(ret == ENOTDIR) return 0; |
---|
[96f002c1] | 139 | |
---|
| 140 | /* paranoid */ verifyf(count == max + 1, "Inconsistent %s count, counted %d, but max %s was %d", prefix, count, prefix, (int)max); |
---|
| 141 | |
---|
| 142 | return count; |
---|
| 143 | } |
---|
| 144 | |
---|
| 145 | // Count number of cpus in the system |
---|
| 146 | static int count_cpus(void) { |
---|
[ab4a595] | 147 | const char * fpath = "/sys/devices/system/cpu/online"; |
---|
[96f002c1] | 148 | int fd = open(fpath, 0, O_RDONLY); |
---|
| 149 | /* paranoid */ verifyf(fd >= 0, "Could not open file %s", fpath); |
---|
| 150 | |
---|
| 151 | char buff[128]; |
---|
| 152 | ssize_t r = read(fd, buff, 128); |
---|
| 153 | /* paranoid */ verifyf(r > 0, "Could not read file %s", fpath); |
---|
| 154 | /* paranoid */ verify( buff[r-1] == '\n' ); |
---|
| 155 | buff[r-1] = '\0'; |
---|
| 156 | |
---|
| 157 | /* paranoid */ __attribute__((unused)) int ret = |
---|
| 158 | close(fd); |
---|
| 159 | /* paranoid */ verifyf(ret == 0, "Could not close file %s", fpath); |
---|
| 160 | |
---|
| 161 | const char * _; |
---|
[ddd2ec9] | 162 | return read_width(buff, r - 1, &_);; |
---|
[96f002c1] | 163 | } |
---|
| 164 | |
---|
| 165 | // Count number of cache *indexes* in the system |
---|
| 166 | // cache indexes are distinct from cache level as Data or Instruction cache |
---|
| 167 | // can share a level but not an index |
---|
| 168 | // PITFALL: assumes all cpus have the same indexes as cpu0 |
---|
| 169 | static int count_cache_indexes(void) { |
---|
| 170 | return count_prefix_dirs("/sys/devices/system/cpu/cpu0/cache", "index"); |
---|
| 171 | } |
---|
| 172 | |
---|
| 173 | |
---|
| 174 | // read information about a spcficic cache index/cpu file into the output buffer |
---|
| 175 | static size_t read_cpuidxinfo_into(unsigned cpu, unsigned idx, const char * file, char * out, size_t out_len) { |
---|
| 176 | // Pick the file we want and read it |
---|
| 177 | char buf[128]; |
---|
| 178 | /* paranoid */ __attribute__((unused)) int len = |
---|
| 179 | snprintf(buf, 128, "/sys/devices/system/cpu/cpu%u/cache/index%u/%s", cpu, idx, file); |
---|
| 180 | /* paranoid */ verifyf(len > 0, "Could not generate '%s' filename for cpu %u, index %u", file, cpu, idx); |
---|
| 181 | |
---|
| 182 | int fd = open(buf, 0, O_RDONLY); |
---|
| 183 | /* paranoid */ verifyf(fd > 0, "Could not open file '%s'", buf); |
---|
| 184 | |
---|
| 185 | ssize_t r = read(fd, out, out_len); |
---|
| 186 | /* paranoid */ verifyf(r > 0, "Could not read file '%s'", buf); |
---|
| 187 | |
---|
| 188 | /* paranoid */ __attribute__((unused)) int ret = |
---|
| 189 | close(fd); |
---|
| 190 | /* paranoid */ verifyf(ret == 0, "Could not close file '%s'", buf); |
---|
| 191 | return r; |
---|
| 192 | } |
---|
| 193 | |
---|
| 194 | // Iterate over the cache indexes of a given cpu |
---|
| 195 | typedef void (*handle_func_t)(unsigned idx, unsigned char level, idx_range_t range, size_t len); |
---|
| 196 | static void foreach_cacheidx(unsigned cpu, unsigned idxs, handle_func_t handle) { |
---|
| 197 | for(i; idxs) { |
---|
| 198 | unsigned idx = idxs - 1 - i; |
---|
| 199 | char buf[32]; |
---|
| 200 | |
---|
| 201 | // Type says what kind of cache this is, |
---|
| 202 | // Options are: Unified, Data, Instruction |
---|
| 203 | read_cpuidxinfo_into(cpu, idx, "type", buf, 32); |
---|
| 204 | if((!strmatch("Unified", buf)) && (!strmatch("Data", buf))) { |
---|
| 205 | // We don't care about instruction caches |
---|
| 206 | continue; |
---|
| 207 | } |
---|
| 208 | |
---|
| 209 | // Level is the cache level: higher means bigger and slower |
---|
| 210 | read_cpuidxinfo_into(cpu, idx, "level", buf, 32); |
---|
| 211 | char * end; |
---|
| 212 | unsigned long level = strtoul(buf, &end, 10); |
---|
| 213 | /* paranoid */ verifyf(level <= 250, "Cpu %u has more than 250 levels of cache, this is not supported", cpu); |
---|
| 214 | |
---|
| 215 | // shared_cpu_list is a range of cpus that share this particular cache |
---|
| 216 | size_t n = read_cpuidxinfo_into(cpu, idx, "shared_cpu_list", buf, 32); |
---|
| 217 | /* paranoid */ verify( buf[n-1] == '\n' ); |
---|
| 218 | buf[n-1] = '\0'; |
---|
| 219 | |
---|
| 220 | // Simply call the functor |
---|
| 221 | handle(idx, level, buf, n - 1); |
---|
| 222 | } |
---|
| 223 | } |
---|
| 224 | |
---|
| 225 | |
---|
| 226 | struct raw_cache_instance { |
---|
[8157bde] | 227 | idx_range_t range; // A text description of the cpus covered |
---|
| 228 | unsigned width; // The number of cpus covered |
---|
| 229 | unsigned char level; // the cache level |
---|
[96f002c1] | 230 | // FIXME add at least size and type |
---|
| 231 | }; |
---|
| 232 | |
---|
| 233 | static void ?{}(raw_cache_instance & this) { this.range = 0p;} |
---|
| 234 | static void ^?{}(raw_cache_instance & this) { free(this.range);} |
---|
| 235 | |
---|
[8157bde] | 236 | // Returns a 2D array of instances of size [cpu count][cache levels] |
---|
| 237 | // where cache level doesn't include instruction caches |
---|
[96f002c1] | 238 | raw_cache_instance ** build_raw_cache_table(unsigned cpus, unsigned idxs, unsigned cache_levels) |
---|
| 239 | { |
---|
| 240 | raw_cache_instance ** raw = alloc(cpus); |
---|
[ddd2ec9] | 241 | |
---|
| 242 | // TODO: this loop is broken, it only works if the present cpu start at 0 and are contiguous which is not guaranteed. |
---|
[96f002c1] | 243 | for(i; cpus) { |
---|
[8157bde] | 244 | if (cache_levels > 0) { |
---|
| 245 | raw[i] = alloc(cache_levels); |
---|
| 246 | void addcache(unsigned fidx, unsigned char level, idx_range_t range, size_t len) { |
---|
| 247 | /* paranoid */ verifyf(level <= cache_levels, "Unexpected cache level %d on cpu %u index %u", (int)level, i, fidx); |
---|
| 248 | |
---|
| 249 | unsigned idx = cache_levels - level; |
---|
| 250 | raw_cache_instance & r = raw[i][idx]; |
---|
| 251 | r.range = strndup(range, len); |
---|
| 252 | r.level = level; |
---|
| 253 | const char * end; |
---|
| 254 | r.width = read_width(range, len, &end); |
---|
| 255 | } |
---|
| 256 | foreach_cacheidx(i, idxs, addcache); |
---|
| 257 | } |
---|
| 258 | else { |
---|
| 259 | char buf[128]; |
---|
| 260 | snprintf(buf, 128, "0-%u", cpus); |
---|
| 261 | raw[i] = alloc(); |
---|
| 262 | raw[i]->range = strndup(buf, 128); |
---|
| 263 | raw[i]->level = 0; |
---|
| 264 | raw[i]->width = cpus; |
---|
[96f002c1] | 265 | } |
---|
| 266 | } |
---|
| 267 | |
---|
| 268 | return raw; |
---|
| 269 | } |
---|
| 270 | |
---|
[cf85f96] | 271 | struct llc_map_t { |
---|
| 272 | raw_cache_instance * raw; |
---|
| 273 | unsigned count; |
---|
| 274 | unsigned start; |
---|
| 275 | }; |
---|
| 276 | |
---|
[96f002c1] | 277 | // returns an allocate list of all the different distinct last level caches |
---|
[cf85f96] | 278 | static [*llc_map_t, size_t cnt] distinct_llcs(unsigned cpus, unsigned llc_idx, raw_cache_instance ** raw) { |
---|
[96f002c1] | 279 | // Allocate at least one element |
---|
[cf85f96] | 280 | llc_map_t* ranges = alloc(); |
---|
[96f002c1] | 281 | size_t range_cnt = 1; |
---|
| 282 | |
---|
| 283 | // Initialize with element 0 |
---|
[cf85f96] | 284 | ranges->raw = &raw[0][llc_idx]; |
---|
| 285 | ranges->count = 0; |
---|
| 286 | ranges->start = -1u; |
---|
[96f002c1] | 287 | |
---|
| 288 | // Go over all other cpus |
---|
| 289 | CPU_LOOP: for(i; 1~cpus) { |
---|
| 290 | // Check if the range is already there |
---|
[cf85f96] | 291 | raw_cache_instance * candidate = &raw[i][llc_idx]; |
---|
[96f002c1] | 292 | for(j; range_cnt) { |
---|
[cf85f96] | 293 | llc_map_t & exist = ranges[j]; |
---|
[96f002c1] | 294 | // If the range is already there just jump to the next cpu |
---|
[cf85f96] | 295 | if(0 == strcmp(candidate->range, exist.raw->range)) continue CPU_LOOP; |
---|
[96f002c1] | 296 | } |
---|
| 297 | |
---|
| 298 | // The range wasn't there, added to the list |
---|
| 299 | ranges = alloc(range_cnt + 1, ranges`realloc); |
---|
[cf85f96] | 300 | ranges[range_cnt].raw = candidate; |
---|
| 301 | ranges[range_cnt].count = 0; |
---|
| 302 | ranges[range_cnt].start = -1u; |
---|
[96f002c1] | 303 | range_cnt++; |
---|
| 304 | } |
---|
| 305 | |
---|
| 306 | // return what we have |
---|
| 307 | return [ranges, range_cnt]; |
---|
| 308 | } |
---|
| 309 | |
---|
| 310 | struct cpu_pairing_t { |
---|
| 311 | unsigned cpu; |
---|
[cf85f96] | 312 | unsigned id; |
---|
[96f002c1] | 313 | }; |
---|
| 314 | |
---|
| 315 | int ?<?( cpu_pairing_t lhs, cpu_pairing_t rhs ) { |
---|
[cf85f96] | 316 | return lhs.id < rhs.id; |
---|
[96f002c1] | 317 | } |
---|
| 318 | |
---|
[cf85f96] | 319 | static [[]cpu_pairing_t] get_cpu_pairings(unsigned cpus, raw_cache_instance ** raw, llc_map_t * maps, size_t map_cnt) { |
---|
[96f002c1] | 320 | cpu_pairing_t * pairings = alloc(cpus); |
---|
| 321 | |
---|
| 322 | CPU_LOOP: for(i; cpus) { |
---|
| 323 | pairings[i].cpu = i; |
---|
| 324 | idx_range_t want = raw[i][0].range; |
---|
| 325 | MAP_LOOP: for(j; map_cnt) { |
---|
[cf85f96] | 326 | if(0 != strcmp(want, maps[j].raw->range)) continue MAP_LOOP; |
---|
[96f002c1] | 327 | |
---|
[cf85f96] | 328 | pairings[i].id = j; |
---|
[96f002c1] | 329 | continue CPU_LOOP; |
---|
| 330 | } |
---|
| 331 | |
---|
| 332 | /* paranoid */ verifyf( false, "Cpu %u map doesn't match", i ); |
---|
| 333 | } |
---|
| 334 | |
---|
| 335 | return pairings; |
---|
| 336 | } |
---|
| 337 | |
---|
[cf85f96] | 338 | #include <fstream.hfa> |
---|
| 339 | |
---|
[96f002c1] | 340 | extern "C" { |
---|
| 341 | void __cfaabi_device_startup( void ) { |
---|
| 342 | int cpus = count_cpus(); |
---|
| 343 | int idxs = count_cache_indexes(); |
---|
| 344 | |
---|
| 345 | // Count actual cache levels |
---|
| 346 | unsigned cache_levels = 0; |
---|
| 347 | unsigned llc = 0; |
---|
[8157bde] | 348 | if (idxs != 0) { |
---|
[96f002c1] | 349 | unsigned char prev = -1u; |
---|
| 350 | void first(unsigned idx, unsigned char level, const char * map, size_t len) { |
---|
| 351 | /* paranoid */ verifyf(level < prev, "Index %u of cpu 0 has cache levels out of order: %u then %u", idx, (unsigned)prev, (unsigned)level); |
---|
| 352 | llc = max(llc, level); |
---|
| 353 | prev = level; |
---|
| 354 | cache_levels++; |
---|
| 355 | } |
---|
| 356 | foreach_cacheidx(0, idxs, first); |
---|
| 357 | } |
---|
| 358 | |
---|
| 359 | // Read in raw data |
---|
| 360 | raw_cache_instance ** raw = build_raw_cache_table(cpus, idxs, cache_levels); |
---|
| 361 | |
---|
| 362 | // Find number of distinct cache instances |
---|
[cf85f96] | 363 | llc_map_t * maps; |
---|
[96f002c1] | 364 | size_t map_cnt; |
---|
| 365 | [maps, map_cnt] = distinct_llcs(cpus, cache_levels - llc, raw); |
---|
| 366 | |
---|
[45fde9f] | 367 | #if defined(__CFA_WITH_VERIFY__) |
---|
[cf85f96] | 368 | // Verify that the caches cover the all the cpus |
---|
[45fde9f] | 369 | { |
---|
[cf85f96] | 370 | unsigned width1 = 0; |
---|
| 371 | unsigned width2 = 0; |
---|
[45fde9f] | 372 | for(i; map_cnt) { |
---|
| 373 | const char * _; |
---|
[cf85f96] | 374 | width1 += read_width(maps[i].raw->range, strlen(maps[i].raw->range), &_); |
---|
| 375 | width2 += maps[i].raw->width; |
---|
[45fde9f] | 376 | } |
---|
[cf85f96] | 377 | verify(width1 == cpus); |
---|
| 378 | verify(width2 == cpus); |
---|
[45fde9f] | 379 | } |
---|
| 380 | #endif |
---|
[96f002c1] | 381 | |
---|
| 382 | // Get mappings from cpu to cache instance |
---|
| 383 | cpu_pairing_t * pairings = get_cpu_pairings(cpus, raw, maps, map_cnt); |
---|
| 384 | |
---|
| 385 | // Sort by cache instance |
---|
| 386 | qsort(pairings, cpus); |
---|
| 387 | |
---|
[cf85f96] | 388 | { |
---|
| 389 | unsigned it = 0; |
---|
| 390 | for(i; cpus) { |
---|
| 391 | unsigned llc_id = pairings[i].id; |
---|
| 392 | if(maps[llc_id].start == -1u) { |
---|
| 393 | maps[llc_id].start = it; |
---|
| 394 | it += maps[llc_id].raw->width; |
---|
| 395 | /* paranoid */ verify(maps[llc_id].start < it); |
---|
| 396 | /* paranoid */ verify(it != -1u); |
---|
| 397 | } |
---|
| 398 | } |
---|
| 399 | /* paranoid */ verify(it == cpus); |
---|
| 400 | } |
---|
[96f002c1] | 401 | |
---|
[cf85f96] | 402 | // From the mappings build the actual cpu map we want |
---|
[96f002c1] | 403 | struct cpu_map_entry_t * entries = alloc(cpus); |
---|
| 404 | for(i; cpus) { entries[i].count = 0; } |
---|
| 405 | for(i; cpus) { |
---|
[cf85f96] | 406 | /* paranoid */ verify(pairings[i].id < map_cnt); |
---|
[96f002c1] | 407 | unsigned c = pairings[i].cpu; |
---|
[cf85f96] | 408 | unsigned llc_id = pairings[i].id; |
---|
| 409 | unsigned width = maps[llc_id].raw->width; |
---|
| 410 | unsigned start = maps[llc_id].start; |
---|
| 411 | unsigned self = start + (maps[llc_id].count++); |
---|
| 412 | entries[c].count = width; |
---|
| 413 | entries[c].start = start; |
---|
| 414 | entries[c].self = self; |
---|
[96f002c1] | 415 | } |
---|
| 416 | |
---|
| 417 | // get rid of the temporary data |
---|
| 418 | free(maps); |
---|
| 419 | free(pairings); |
---|
| 420 | |
---|
| 421 | for(i; cpus) { |
---|
| 422 | for(j; cache_levels) { |
---|
| 423 | ^(raw[i][j]){}; |
---|
| 424 | } |
---|
| 425 | free(raw[i]); |
---|
| 426 | } |
---|
| 427 | free(raw); |
---|
| 428 | |
---|
| 429 | cpu_info.llc_map = entries; |
---|
| 430 | cpu_info.hthrd_count = cpus; |
---|
[8157bde] | 431 | cpu_info.llc_count = map_cnt; |
---|
[96f002c1] | 432 | } |
---|
| 433 | |
---|
| 434 | void __cfaabi_device_shutdown( void ) { |
---|
| 435 | free(cpu_info.llc_map); |
---|
| 436 | } |
---|
[72bd9cd] | 437 | } |
---|
| 438 | |
---|
| 439 | cpu_info_t cpu_info; |
---|