Find a subset of problems that maximal explains rating.

    Usage:
        python3 sharp_positions.py minimize \
            --model_dir models --sgf_dir data/s
            --rating_json ratings.json --results results.csv

def minimize():
    """Find a subset of problems that maximal explains rating.

    Usage:
        python3 sharp_positions.py minimize \
            --model_dir models --sgf_dir data/s
            --rating_json ratings.json --results results.csv
    """
    ########################### HYPER PARAMETERS ###############################

    # Stop when r2 is this much worse than full set of positions
    r2_stopping_percent = 0.96
    # for this many iterations
    stopping_iterations = 5

    # Limit SVM to a smaller number of positions to speed up code.
    max_positions_fit = 300
    # Filter any position that "contributes" less than this percent of max.
    filter_contribution_percent = 0.3
    # Never filter more than this many positions in one iterations
    filter_limit = 25

    ########################### HYPER PARAMETERS ###############################

    # Load positons
    model_paths = oneoff_utils.get_model_paths(FLAGS.model_dir)
    num_models = len(model_paths)
    assert num_models > 0, FLAGS.model_dir

    # Load model ratings
    # wget https://cloudygo.com/v12-19x19/json/ratings.json
    ratings = json.load(open(FLAGS.rating_json))
    raw_ratings = {int(r[0]): float(r[1]) for r in ratings}

    model_ratings = []
    for model in model_paths:
        model_idx = get_model_idx(model)
        if model_idx < FLAGS.min_idx:
            continue

        model_ratings.append(raw_ratings[model_idx])
    model_ratings = np.array(model_ratings)

    assert 0 < len(model_ratings) <= num_models, len(model_ratings)
    num_models = len(model_ratings)

    sgf_names, all_positions = get_final_positions()
    # Trim off common path prefix.
    common_path = os.path.commonpath(sgf_names)
    sgf_names = [name[len(common_path) + 1:] for name in sgf_names]

    print("Considering {} positions, {} models".format(
        len(all_positions), num_models))
    print()

    # Load model data
    top_n = FLAGS.top_n
    positions = defaultdict(list)
    with open(FLAGS.results) as results:
        headers = results.readline().strip()
        assert headers.count(",") + 1 == len(sgf_names)

        # Row is <model_name> + positions x [value, top_n x [move, move_policy]]
        for row in tqdm(results.readlines(), desc="result line"):
            data = row.split(",")
            model_idx = get_model_idx(data.pop(0))
            if model_idx < FLAGS.min_idx:
                continue

            data_per = 1 + top_n * 2
            assert len(data) % data_per == 0, len(data)

            for position, position_data in enumerate(grouper(data_per, data)):
                value = float(position_data.pop(0))
                moves = list(map(int, position_data[0::2]))
                move_policy = list(map(float, position_data[1::2]))

                positions[position].append([value, moves, move_policy])

    def one_hot(n, i):
        one_hot = [0] * n
        if 0 <= i < n:
            one_hot[i] += 1
        return one_hot

    # NOTE: top_n isn't the same semantic value here and can be increased.
    one_hot_moves = top_n
    num_features = 1 + 5 + (one_hot_moves + 1)

    # Features by position
    features = []
    pos_top_moves = []
    for position, data in tqdm(positions.items(), desc="featurize"):
        assert len(data) == num_models, len(data)

        top_moves = Counter([d[1][0] for d in data])
        top_n_moves = [m for m, c in top_moves.most_common(one_hot_moves)]
        if len(top_n_moves) < one_hot_moves:
            top_n_moves.extend([-1] * (one_hot_moves - len(top_n_moves)))
        assert len(top_n_moves) == one_hot_moves, "pad with dummy moves"
        pos_top_moves.append(top_n_moves)

        # Eventaully we want
        # [model 1 position 1 features, m1 p2 features, m1 p3 features, ... ]
        # [model 2 position 1 features, m2 p2 features, m2 p3 features, ... ]
        # [model 3 position 1 features, m3 p2 features, m3 p3 features, ... ]
        # ...
        # [model m position 1 features, mm p2 features, mm p3 features, ... ]

        # We'll do position selection by joining [model x position_feature]

        feature_columns = []
        for model, (v, m, mv) in enumerate(data):
            # Featurization (for each positions):
            #   * Value (-1 to 1), Bucketed value
            #   * Cluster all model by top_n moves (X,Y,Z or other)?
            #     * value of that move for model
            #   * policy value of top move
            model_features = []

            model_features.append(2 * v - 1)
            # NOTE(sethtroisi): Consider bucketize value by value percentiles.
            value_bucket = np.searchsorted((0.2, 0.4, 0.6, 0.8), v)
            model_features.extend(one_hot(5, value_bucket))

            # Policy weight for most common X moves (among all models).
            policy_weights = [0] * (one_hot_moves + 1)
            for move, policy_value in zip(m, mv):
                if move in top_n_moves:
                    policy_weights[top_n_moves.index(move)] = policy_value
                else:
                    policy_weights[-1] += policy_value
            model_features.extend(policy_weights)

            assert len(model_features) == num_features

            feature_columns.append(model_features)
        features.append(feature_columns)

    features = np.array(features)
    print("Feature shape", features.shape)
    print()

    # Split the models to test / train
    train_size = int(num_models * 0.9)
    train_models = sorted(np.random.permutation(num_models)[:train_size])
    test_models = sorted(set(range(num_models)) - set(train_models))
    assert set(train_models + test_models) == set(range(num_models))
    features_train = features[:, train_models, :]
    features_test  = features[:, test_models, :]

    labels_train = model_ratings[train_models]
    labels_test = model_ratings[test_models]

    # Choose some set of positions and see how well they explain ratings
    positions_to_use = set(positions.keys())
    linearSVM = svm.LinearSVR()
    best_test_r2 = 0
    below_threshold = 0

    for iteration in itertools.count(1):
        iter_positions = np.random.permutation(list(positions_to_use))
        iter_positions = sorted(iter_positions[:max_positions_fit])

        # Take this set of positions and build X
        X = np.concatenate(features_train[iter_positions], axis=1)
        Xtest = np.concatenate(features_test[iter_positions], axis=1)
        assert X.shape == (train_size, num_features * len(iter_positions))

        linearSVM.fit(X, labels_train)

        score_train = linearSVM.score(X, labels_train)
        score_test = linearSVM.score(Xtest, labels_test)
        print("iter {}, {}/{} included, R^2: {:.4f} train, {:.3f} test".format(
            iteration, len(iter_positions), len(positions_to_use),
            score_train, score_test))

        # Determine the most and least useful position:
        # TODO(amj,brilee): Validate this math.
        assert len(linearSVM.coef_) == num_features * len(iter_positions)

        # The intercepts tell us how much this contributes to overall rating
        # but coef tell us how much different answers differentiate rating.
        coef_groups = list(grouper(num_features, linearSVM.coef_))
        position_coefs = [abs(sum(c)) for c in coef_groups]

        pos_value_idx = np.argsort(position_coefs)
        max_pos = pos_value_idx[-1]
        most_value = position_coefs[max_pos]

        print("\tMost value {} => {:.1f} {}".format(
            max_pos, most_value, sgf_names[iter_positions[max_pos]]))

        # Drop any positions that aren't very useful
        for dropped, pos_idx in enumerate(pos_value_idx[:filter_limit], 1):
            contribution = position_coefs[pos_idx]
            positions_to_use.remove(iter_positions[pos_idx])
            print("\t\tdropping({}): {:.1f} {}".format(
                dropped, contribution, sgf_names[iter_positions[pos_idx]]))

            if contribution > filter_contribution_percent * most_value:
                break
        print()

        best_test_r2 = max(best_test_r2, score_test)
        if score_test > r2_stopping_percent * best_test_r2:
            below_threshold = 0
        else:
            below_threshold += 1
            if below_threshold == stopping_iterations:
                print("{}% decrease in R^2, stopping".format(
                    100 - int(100 * r2_stopping_percent)))
                break

    # Write down the differentiating positions and their answers.
    svm_data = []
    for position_idx in list(reversed(pos_value_idx)):
        coefs = coef_groups[position_idx]

        # Global position index.
        position = iter_positions[position_idx]
        sgf_name = sgf_names[position]
        top_moves = pos_top_moves[position]

        svm_data.append([sgf_name, [top_moves, coefs.tolist()]])

    with open(FLAGS.SVM_json, "w") as svm_json:
        json.dump(svm_data, svm_json)
    print("Dumped data about {} positions to {}".format(
        len(svm_data), FLAGS.SVM_json))