source: benchmark/rmit.py @ 726b748

#!/usr/bin/python3
"""
Python Script to implement R.M.I.T. testing : Randomized Multiple Interleaved Trials

./rmit.py run COMMAND CANDIDATES
-t trials
-o option:values
"""


import argparse
import datetime
import itertools
import json
import os
import random
import re
import subprocess
import sys


def parse_range(x):
    result = []
    for part in x.split(','):
        if '-' in part:
            a, b = part.split('-')
            a, b = int(a), int(b)
            result.extend(range(a, b + 1))
        else:
            a = int(part)
            result.append(a)
    return result

class DependentOpt:
        def __init__(self, key, value):
                self.key = key
                self.value = value
                self.vars = re.findall("[a-zA-Z]", value)

def parse_option(opt):
        match = re.search("^(.*):(.*)$", opt)
        if not match :
                print('ERROR: extra options should match pattern .*:.*, got {}'.format(opt), file=sys.stderr)
                sys.exit(1)

        key    = match.group(1)
        values = match.group(2)

        try:
                num = int(values)
                return key, [num]
        except:
                pass

        if re.search("^[0-9-,]+$", values):
                values = parse_range(values)
                return key, [v for v in values]
        else:
                return key, DependentOpt(key, values)

def eval_one(fmt, vals):
        orig = fmt
        for k, v in vals:
                fmt = fmt.replace(k, str(v))

        if not re.search("^[0-9-/*+ ]+$", fmt):
                print('ERROR: pattern option {} (interpreted as {}) could not be evaluated'.format(orig, fmt), file=sys.stderr)
                sys.exit(1)

        return eval(fmt)

def eval_options(opts):
        dependents = [d for d in opts.values() if type(d) is DependentOpt]
        processed = []
        nopts = []
        for d in dependents:
                processed.append(d.key)
                lists = []
                for dvar in d.vars:
                        if not dvar in opts.keys():
                                print('ERROR: extra pattern option {}:{} uses unknown key {}'.format(d.key,d.value,dvar), file=sys.stderr)
                                sys.exit(1)

                        lists.append([(dvar, o) for o in opts[dvar]])
                        processed.append(dvar)

                kopt = []
                for vals in list(itertools.product(*lists)):
                        res = ['-{}'.format(d.key), "{}".format(eval_one(d.value, vals))]
                        for k, v in vals:
                                res.extend(['-{}'.format(k), "{}".format(v)])
                        kopt.append(res)
                nopts.append(kopt)


        for k, vals in opts.items():
                if k not in processed:
                        kopt = []
                        for v in vals:
                                kopt.append(['-{}'.format(k), "{}".format(v)])
                        nopts.append(kopt)

        return nopts

def actions_eta(actions):
        time = 0
        for a in actions:
                i = 0
                while i < len(a):
                        if a[i] == '-d':
                                i += 1
                                if i != len(a):
                                        time += int(a[i])
                        i += 1
        return time

if __name__ == "__main__":
        # ================================================================================
        # parse command line arguments
        formats = ['raw', 'csv', 'json']
        parser = argparse.ArgumentParser(description='Python Script to implement R.M.I.T. testing : Randomized Multiple Interleaved Trials')
        parser.add_argument('--list', help='List all the commands that would be run', action='store_true')
        parser.add_argument('--format', help='How to print the result', choices=formats, default='json')
        parser.add_argument('--file', nargs='?', type=argparse.FileType('w'), default=sys.stdout)
        parser.add_argument('-t', '--trials', help='Number of trials to run per combinaison', type=int, default=3)
        parser.add_argument('-o','--option',action='append')
        parser.add_argument('command', metavar='command', type=str, nargs=1, help='the command prefix to run')
        parser.add_argument('candidates', metavar='candidates', type=str, nargs='*', help='the candidate suffix to run')

        try:
                options =  parser.parse_args()
        except:
                print('ERROR: invalid arguments', file=sys.stderr)
                parser.print_help(sys.stderr)
                sys.exit(1)

        # ================================================================================
        # Identify the commands to run
        commands = ["./" + options.command[0] + "-" + c for c in options.candidates]
        for c in commands:
                if not os.path.isfile(c):
                        print('ERROR: invalid command {}, file does not exist'.format(c), file=sys.stderr)
                        sys.exit(1)

                if not os.access(c, os.X_OK):
                        print('ERROR: invalid command {}, file not executable'.format(c), file=sys.stderr)
                        sys.exit(1)


        # ================================================================================
        # Identify the options to run
        opts = dict([parse_option(o) for o in options.option])

        # Evaluate the options (options can depend on the value of other options)
        opts = eval_options(opts)

        # ================================================================================
        # Figure out all the combinations to run
        actions = []
        for p in itertools.product(range(options.trials), commands, *opts):
                act = [p[1]]
                for o in p[2:]:
                        act.extend(o)
                actions.append(act)

        # ================================================================================
        # Figure out all the combinations to run
        if options.list:
                for a in actions:
                        print(" ".join(a))
                sys.exit(0)


        # ================================================================================
        # Prepare to run

        # find expected time
        time = actions_eta(actions)
        print("Running {} trials{}".format(len(actions), "" if time == 0 else " (expecting to take {}".format(str(datetime.timedelta(seconds=int(time)))) ))

        random.shuffle(actions)
        result = []

        # ================================================================================
        # Run
        for i, a in enumerate(actions):
                sa = " ".join(a)
                print("{}/{} : {}          \r".format(i, len(actions), sa), end = '')
                fields = {}
                with subprocess.Popen( a, stdout  = subprocess.PIPE, stderr  = subprocess.PIPE) as proc:
                        out, err = proc.communicate()
                        if proc.returncode != 0:
                                print("ERROR: command '{}' encountered error, returned code {}".format(sa, proc.returncode), file=sys.stderr)
                                print(err.decode("utf-8"))
                                sys.exit(1)
                        for s in out.decode("utf-8").splitlines():
                                match = re.search("^(.*):(.*)$", s)
                                if match:
                                        fields[match.group(1).strip()] = float(match.group(2).strip().replace(',',''))

                result.append([a[0][2:], sa, fields])

        print("Done                                                                                ")

        # ================================================================================
        # Print raw
        if options.format == 'raw':
                for r in result:
                        print(r, file=options.file)
                sys.exit(0)

        # ================================================================================
        # Print json
        if options.format == 'json':
                json.dump(result, options.file)
                sys.exit(0)

        # ================================================================================
        # Print csv
        if options.format == 'csv':
                # Clean result
                headers = ["series", "command"]
                data = []
                first = True
                for r in result:
                        if first:
                                first = False
                                headers.extend(r[2].keys())
                        else :
                                pass

                        d = [r[0], r[1]]
                        for k in headers[2:]:
                                try:
                                        d.append(r[2][k])
                                except:
                                        d.append(0.0)

                        data.append(d)

                # Print csv
                print(",\t".join(headers), file=options.file)
                for d in data:
                        print(",\t".join(["{}".format(dd) for dd in d]), file=options.file)
                sys.exit(0)