@borg.annotations(
    out_path = ("results CSV output path"),
    runs = ("path to JSON runs specification", "positional", None, borg.util.load_json),
    repeats = ("number of times to repeat each run", "option", None, int),
    workers = ("submit jobs?", "option", "w", int),
    local = ("workers are local?", "flag"),
    )
def main(out_path, runs, repeats = 5, workers = 0, local = False):

    logger.info("simulating %i runs", len(runs) * repeats)

    get_run_data = borg.util.memoize(borg.storage.RunData.from_bundle)

    def yield_jobs():
        for run in runs:
            train_data = get_run_data(run["train_bundle"])

            if run.get("only_nontrivial", False):
                train_data = train_data.only_nontrivial()

            if run["test_bundle"] == "-":
                validation = sklearn.cross_validation.KFold(len(train_data), repeats, indices = False)
                data_sets = [(train_data.masked(v), train_data.masked(e)) for (v, e) in validation]
            else:
                test_data = get_run_data(run["test_bundle"])

                if run.get("only_nontrivial", False):
                    test_data = test_data.only_nontrivial()

                data_sets = [(train_data, test_data)] * repeats

            if run["portfolio_name"] == "-":
                makers = map(SolverMaker, train_data.solver_names)
            else:
                makers = [PortfolioMaker(run["portfolio_name"])]

            for maker in makers:
                for (train_fold_data, test_fold_data) in data_sets:
                    yield (simulate_run, [run, maker, train_fold_data, test_fold_data])

    with borg.util.openz(out_path, "wb") as out_file:
        writer = csv.writer(out_file)

        writer.writerow(["category", "solver", "budget", "cost", "success", "split"])

        for (_, rows) in condor.do(yield_jobs(), workers, local):
            writer.writerows(rows)

            out_file.flush()
Simulate portfolio and solver behavior.
dataset/ETHPy150Open borg-project/borg/borg/experiments/simulate_runs.py/main