|
Last change
on this file since adaf062 was 0a79fd9, checked in by Thierry Delisle <tdelisle@…>, 4 years ago |
|
Simple python tool to down sample data
|
-
Property mode
set to
100755
|
|
File size:
670 bytes
|
| Line | |
|---|
| 1 | #!/usr/bin/python3
|
|---|
| 2 |
|
|---|
| 3 | import argparse, json, random, sys
|
|---|
| 4 |
|
|---|
| 5 | parser = argparse.ArgumentParser()
|
|---|
| 6 | parser.add_argument('--infile', type=argparse.FileType('r'), default=sys.stdin)
|
|---|
| 7 | parser.add_argument('--outfile', type=argparse.FileType('w'), default=sys.stdout)
|
|---|
| 8 |
|
|---|
| 9 | args = parser.parse_args()
|
|---|
| 10 |
|
|---|
| 11 | data = json.load(args.infile)
|
|---|
| 12 |
|
|---|
| 13 |
|
|---|
| 14 |
|
|---|
| 15 | print(len(data['values']))
|
|---|
| 16 | print(int(len(data['values']) / 1000))
|
|---|
| 17 |
|
|---|
| 18 | sample = random.sample(data['values'], int(len(data['values']) / 1000))
|
|---|
| 19 | print(len(sample))
|
|---|
| 20 |
|
|---|
| 21 | # Sort by timestamp (the second element)
|
|---|
| 22 | # take second element for sort
|
|---|
| 23 | def takeSecond(elem):
|
|---|
| 24 | return elem[1]
|
|---|
| 25 |
|
|---|
| 26 | sample.sort(key=takeSecond)
|
|---|
| 27 |
|
|---|
| 28 | data['values'] = sample
|
|---|
| 29 | json.dump(data, args.outfile)
|
|---|
Note:
See
TracBrowser
for help on using the repository browser.
