Last change
on this file since 5b643ea was
0a79fd9,
checked in by Thierry Delisle <tdelisle@…>, 3 years ago
|
Simple python tool to down sample data
|
-
Property mode set to
100755
|
File size:
670 bytes
|
Rev | Line | |
---|
[0a79fd9] | 1 | #!/usr/bin/python3 |
---|
| 2 | |
---|
| 3 | import argparse, json, random, sys |
---|
| 4 | |
---|
| 5 | parser = argparse.ArgumentParser() |
---|
| 6 | parser.add_argument('--infile', type=argparse.FileType('r'), default=sys.stdin) |
---|
| 7 | parser.add_argument('--outfile', type=argparse.FileType('w'), default=sys.stdout) |
---|
| 8 | |
---|
| 9 | args = parser.parse_args() |
---|
| 10 | |
---|
| 11 | data = json.load(args.infile) |
---|
| 12 | |
---|
| 13 | |
---|
| 14 | |
---|
| 15 | print(len(data['values'])) |
---|
| 16 | print(int(len(data['values']) / 1000)) |
---|
| 17 | |
---|
| 18 | sample = random.sample(data['values'], int(len(data['values']) / 1000)) |
---|
| 19 | print(len(sample)) |
---|
| 20 | |
---|
| 21 | # Sort by timestamp (the second element) |
---|
| 22 | # take second element for sort |
---|
| 23 | def takeSecond(elem): |
---|
| 24 | return elem[1] |
---|
| 25 | |
---|
| 26 | sample.sort(key=takeSecond) |
---|
| 27 | |
---|
| 28 | data['values'] = sample |
---|
| 29 | json.dump(data, args.outfile) |
---|
Note: See
TracBrowser
for help on using the repository browser.