def minimize():
    """Find a subset of problems that maximal explains rating.

    Usage:
        python3 sharp_positions.py minimize \
            --model_dir models --sgf_dir data/s
            --rating_json ratings.json --results results.csv
    """
    ########################### HYPER PARAMETERS ###############################

    # Stop when r2 is this much worse than full set of positions
    r2_stopping_percent = 0.96
    # for this many iterations
    stopping_iterations = 5

    # Limit SVM to a smaller number of positions to speed up code.
    max_positions_fit = 300
    # Filter any position that "contributes" less than this percent of max.
    filter_contribution_percent = 0.3
    # Never filter more than this many positions in one iterations
    filter_limit = 25

    ########################### HYPER PARAMETERS ###############################

    # Load positons
    model_paths = oneoff_utils.get_model_paths(FLAGS.model_dir)
    num_models = len(model_paths)
    assert num_models > 0, FLAGS.model_dir

    # Load model ratings
    # wget https://cloudygo.com/v12-19x19/json/ratings.json
    ratings = json.load(open(FLAGS.rating_json))
    raw_ratings = {int(r[0]): float(r[1]) for r in ratings}

    model_ratings = []
    for model in model_paths:
        model_idx = get_model_idx(model)
        if model_idx < FLAGS.min_idx:
            continue

        model_ratings.append(raw_ratings[model_idx])
    model_ratings = np.array(model_ratings)

    assert 0 < len(model_ratings) <= num_models, len(model_ratings)
    num_models = len(model_ratings)

    sgf_names, all_positions = get_final_positions()
    # Trim off common path prefix.
    common_path = os.path.commonpath(sgf_names)
    sgf_names = [name[len(common_path) + 1:] for name in sgf_names]

    print("Considering {} positions, {} models".format(
        len(all_positions), num_models))
    print()

    # Load model data
    top_n = FLAGS.top_n
    positions = defaultdict(list)
    with open(FLAGS.results) as results:
        headers = results.readline().strip()
        assert headers.count(",") + 1 == len(sgf_names)

        # Row is <model_name> + positions x [value, top_n x [move, move_policy]]
        for row in tqdm(results.readlines(), desc="result line"):
            data = row.split(",")
            model_idx = get_model_idx(data.pop(0))
            if model_idx < FLAGS.min_idx:
                continue

            data_per = 1 + top_n * 2
            assert len(data) % data_per == 0, len(data)

            for position, position_data in enumerate(grouper(data_per, data)):
                value = float(position_data.pop(0))
                moves = list(map(int, position_data[0::2]))
                move_policy = list(map(float, position_data[1::2]))

                positions[position].append([value, moves, move_policy])

    def one_hot(n, i):
        one_hot = [0] * n
        if 0 <= i < n:
            one_hot[i] += 1
        return one_hot

    # NOTE: top_n isn't the same semantic value here and can be increased.
    one_hot_moves = top_n
    num_features = 1 + 5 + (one_hot_moves + 1)

    # Features by position
    features = []
    pos_top_moves = []
    for position, data in tqdm(positions.items(), desc="featurize"):
        assert len(data) == num_models, len(data)

        top_moves = Counter([d[1][0] for d in data])
        top_n_moves = [m for m, c in top_moves.most_common(one_hot_moves)]
        if len(top_n_moves) < one_hot_moves:
            top_n_moves.extend([-1] * (one_hot_moves - len(top_n_moves)))
        assert len(top_n_moves) == one_hot_moves, "pad with dummy moves"
        pos_top_moves.append(top_n_moves)

        # Eventaully we want
        # [model 1 position 1 features, m1 p2 features, m1 p3 features, ... ]
        # [model 2 position 1 features, m2 p2 features, m2 p3 features, ... ]
        # [model 3 position 1 features, m3 p2 features, m3 p3 features, ... ]
        # ...
        # [model m position 1 features, mm p2 features, mm p3 features, ... ]

        # We'll do position selection by joining [model x position_feature]

        feature_columns = []
        for model, (v, m, mv) in enumerate(data):
            # Featurization (for each positions):
            #   * Value (-1 to 1), Bucketed value
            #   * Cluster all model by top_n moves (X,Y,Z or other)?
            #     * value of that move for model
            #   * policy value of top move
            model_features = []

            model_features.append(2 * v - 1)
            # NOTE(sethtroisi): Consider bucketize value by value percentiles.
            value_bucket = np.searchsorted((0.2, 0.4, 0.6, 0.8), v)
            model_features.extend(one_hot(5, value_bucket))

            # Policy weight for most common X moves (among all models).
            policy_weights = [0] * (one_hot_moves + 1)
            for move, policy_value in zip(m, mv):
                if move in top_n_moves:
                    policy_weights[top_n_moves.index(move)] = policy_value
                else:
                    policy_weights[-1] += policy_value
            model_features.extend(policy_weights)

            assert len(model_features) == num_features

            feature_columns.append(model_features)
        features.append(feature_columns)

    features = np.array(features)
    print("Feature shape", features.shape)
    print()

    # Split the models to test / train
    train_size = int(num_models * 0.9)
    train_models = sorted(np.random.permutation(num_models)[:train_size])
    test_models = sorted(set(range(num_models)) - set(train_models))
    assert set(train_models + test_models) == set(range(num_models))
    features_train = features[:, train_models, :]
    features_test  = features[:, test_models, :]

    labels_train = model_ratings[train_models]
    labels_test = model_ratings[test_models]

    # Choose some set of positions and see how well they explain ratings
    positions_to_use = set(positions.keys())
    linearSVM = svm.LinearSVR()
    best_test_r2 = 0
    below_threshold = 0

    for iteration in itertools.count(1):
        iter_positions = np.random.permutation(list(positions_to_use))
        iter_positions = sorted(iter_positions[:max_positions_fit])

        # Take this set of positions and build X
        X = np.concatenate(features_train[iter_positions], axis=1)
        Xtest = np.concatenate(features_test[iter_positions], axis=1)
        assert X.shape == (train_size, num_features * len(iter_positions))

        linearSVM.fit(X, labels_train)

        score_train = linearSVM.score(X, labels_train)
        score_test = linearSVM.score(Xtest, labels_test)
        print("iter {}, {}/{} included, R^2: {:.4f} train, {:.3f} test".format(
            iteration, len(iter_positions), len(positions_to_use),
            score_train, score_test))

        # Determine the most and least useful position:
        # TODO(amj,brilee): Validate this math.
        assert len(linearSVM.coef_) == num_features * len(iter_positions)

        # The intercepts tell us how much this contributes to overall rating
        # but coef tell us how much different answers differentiate rating.
        coef_groups = list(grouper(num_features, linearSVM.coef_))
        position_coefs = [abs(sum(c)) for c in coef_groups]

        pos_value_idx = np.argsort(position_coefs)
        max_pos = pos_value_idx[-1]
        most_value = position_coefs[max_pos]

        print("\tMost value {} => {:.1f} {}".format(
            max_pos, most_value, sgf_names[iter_positions[max_pos]]))

        # Drop any positions that aren't very useful
        for dropped, pos_idx in enumerate(pos_value_idx[:filter_limit], 1):
            contribution = position_coefs[pos_idx]
            positions_to_use.remove(iter_positions[pos_idx])
            print("\t\tdropping({}): {:.1f} {}".format(
                dropped, contribution, sgf_names[iter_positions[pos_idx]]))

            if contribution > filter_contribution_percent * most_value:
                break
        print()

        best_test_r2 = max(best_test_r2, score_test)
        if score_test > r2_stopping_percent * best_test_r2:
            below_threshold = 0
        else:
            below_threshold += 1
            if below_threshold == stopping_iterations:
                print("{}% decrease in R^2, stopping".format(
                    100 - int(100 * r2_stopping_percent)))
                break

    # Write down the differentiating positions and their answers.
    svm_data = []
    for position_idx in list(reversed(pos_value_idx)):
        coefs = coef_groups[position_idx]

        # Global position index.
        position = iter_positions[position_idx]
        sgf_name = sgf_names[position]
        top_moves = pos_top_moves[position]

        svm_data.append([sgf_name, [top_moves, coefs.tolist()]])

    with open(FLAGS.SVM_json, "w") as svm_json:
        json.dump(svm_data, svm_json)
    print("Dumped data about {} positions to {}".format(
        len(svm_data), FLAGS.SVM_json))
def minimize ( ) : r2_stopping_percent = 0.96 stopping_iterations = 5 max_positions_fit = 300 filter_contribution_percent = 0.3 filter_limit = 25 model_paths = oneoff_utils . get_model_paths ( FLAGS . model_dir ) num_models = len ( model_paths ) assert num_models > 0 , FLAGS . model_dir ratings = json . load ( open ( FLAGS . rating_json ) ) raw_ratings = { int ( r [ 0 ] ) : float ( r [ 1 ] ) for r in ratings } model_ratings = [ ] for model in model_paths : model_idx = get_model_idx ( model ) if model_idx < FLAGS . min_idx : continue model_ratings . append ( raw_ratings [ model_idx ] ) model_ratings = np . array ( model_ratings ) assert 0 < len ( model_ratings ) <= num_models , len ( model_ratings ) num_models = len ( model_ratings ) sgf_names , all_positions = get_final_positions ( ) common_path = os . path . commonpath ( sgf_names ) sgf_names = [ name [ len ( common_path ) + 1 : ] for name in sgf_names ] print ( "Considering {} positions, {} models" . format ( len ( all_positions ) , num_models ) ) print ( ) top_n = FLAGS . top_n positions = defaultdict ( list ) with open ( FLAGS . results ) as results : headers = results . readline ( ) . strip ( ) assert headers . count ( "," ) + 1 == len ( sgf_names ) for row in tqdm ( results . readlines ( ) , desc = "result line" ) : data = row . split ( "," ) model_idx = get_model_idx ( data . pop ( 0 ) ) if model_idx < FLAGS . min_idx : continue data_per = 1 + top_n * 2 assert len ( data ) % data_per == 0 , len ( data ) for position , position_data in enumerate ( grouper ( data_per , data ) ) : value = float ( position_data . pop ( 0 ) ) moves = list ( map ( int , position_data [ 0 : : 2 ] ) ) move_policy = list ( map ( float , position_data [ 1 : : 2 ] ) ) positions [ position ] . append ( [ value , moves , move_policy ] ) def one_hot ( n , i ) : one_hot = [ 0 ] * n if 0 <= i < n : one_hot [ i ] += 1 return one_hot one_hot_moves = top_n num_features = 1 + 5 + ( one_hot_moves + 1 ) features = [ ] pos_top_moves = [ ] for position , data in tqdm ( positions . items ( ) , desc = "featurize" ) : assert len ( data ) == num_models , len ( data ) top_moves = Counter ( [ d [ 1 ] [ 0 ] for d in data ] ) top_n_moves = [ m for m , c in top_moves . most_common ( one_hot_moves ) ] if len ( top_n_moves ) < one_hot_moves : top_n_moves . extend ( [ - 1 ] * ( one_hot_moves - len ( top_n_moves ) ) ) assert len ( top_n_moves ) == one_hot_moves , "pad with dummy moves" pos_top_moves . append ( top_n_moves ) feature_columns = [ ] for model , ( v , m , mv ) in enumerate ( data ) : model_features = [ ] model_features . append ( 2 * v - 1 ) value_bucket = np . searchsorted ( ( 0.2 , 0.4 , 0.6 , 0.8 ) , v ) model_features . extend ( one_hot ( 5 , value_bucket ) ) policy_weights = [ 0 ] * ( one_hot_moves + 1 ) for move , policy_value in zip ( m , mv ) : if move in top_n_moves : policy_weights [ top_n_moves . index ( move ) ] = policy_value else : policy_weights [ - 1 ] += policy_value model_features . extend ( policy_weights ) assert len ( model_features ) == num_features feature_columns . append ( model_features ) features . append ( feature_columns ) features = np . array ( features ) print ( "Feature shape" , features . shape ) print ( ) train_size = int ( num_models * 0.9 ) train_models = sorted ( np . random . permutation ( num_models ) [ : train_size ] ) test_models = sorted ( set ( range ( num_models ) ) - set ( train_models ) ) assert set ( train_models + test_models ) == set ( range ( num_models ) ) features_train = features [ : , train_models , : ] features_test = features [ : , test_models , : ] labels_train = model_ratings [ train_models ] labels_test = model_ratings [ test_models ] positions_to_use = set ( positions . keys ( ) ) linearSVM = svm . LinearSVR ( ) best_test_r2 = 0 below_threshold = 0 for iteration in itertools . count ( 1 ) : iter_positions = np . random . permutation ( list ( positions_to_use ) ) iter_positions = sorted ( iter_positions [ : max_positions_fit ] ) X = np . concatenate ( features_train [ iter_positions ] , axis = 1 ) Xtest = np . concatenate ( features_test [ iter_positions ] , axis = 1 ) assert X . shape == ( train_size , num_features * len ( iter_positions ) ) linearSVM . fit ( X , labels_train ) score_train = linearSVM . score ( X , labels_train ) score_test = linearSVM . score ( Xtest , labels_test ) print ( "iter {}, {}/{} included, R^2: {:.4f} train, {:.3f} test" . format ( iteration , len ( iter_positions ) , len ( positions_to_use ) , score_train , score_test ) ) assert len ( linearSVM . coef_ ) == num_features * len ( iter_positions ) coef_groups = list ( grouper ( num_features , linearSVM . coef_ ) ) position_coefs = [ abs ( sum ( c ) ) for c in coef_groups ] pos_value_idx = np . argsort ( position_coefs ) max_pos = pos_value_idx [ - 1 ] most_value = position_coefs [ max_pos ] print ( "\tMost value {} => {:.1f} {}" . format ( max_pos , most_value , sgf_names [ iter_positions [ max_pos ] ] ) ) for dropped , pos_idx in enumerate ( pos_value_idx [ : filter_limit ] , 1 ) : contribution = position_coefs [ pos_idx ] positions_to_use . remove ( iter_positions [ pos_idx ] ) print ( "\t\tdropping({}): {:.1f} {}" . format ( dropped , contribution , sgf_names [ iter_positions [ pos_idx ] ] ) ) if contribution > filter_contribution_percent * most_value : break print ( ) best_test_r2 = max ( best_test_r2 , score_test ) if score_test > r2_stopping_percent * best_test_r2 : below_threshold = 0 else : below_threshold += 1 if below_threshold == stopping_iterations : print ( "{}% decrease in R^2, stopping" . format ( 100 - int ( 100 * r2_stopping_percent ) ) ) break svm_data = [ ] for position_idx in list ( reversed ( pos_value_idx ) ) : coefs = coef_groups [ position_idx ] position = iter_positions [ position_idx ] sgf_name = sgf_names [ position ] top_moves = pos_top_moves [ position ] svm_data . append ( [ sgf_name , [ top_moves , coefs . tolist ( ) ] ] ) with open ( FLAGS . SVM_json , "w" ) as svm_json : json . dump ( svm_data , svm_json ) print ( "Dumped data about {} positions to {}" . format ( len ( svm_data ) , FLAGS . SVM_json ) )
Find a subset of problems that maximal explains rating.

    Usage:
        python3 sharp_positions.py minimize \
            --model_dir models --sgf_dir data/s
            --rating_json ratings.json --results results.csv