<s> """<STR_LIT>""" <EOL> from __future__ import ( absolute_import , division , print_function , <EOL> unicode_literals ) <EOL> import csv <EOL> import itertools <EOL> from io import open <EOL> import os <EOL> from os . path import abspath , dirname , join , exists <EOL> import json <EOL> from glob import glob <EOL> import numpy as np <EOL> from nose . tools import eq_ , raises <EOL> from six import PY2 <EOL> import random <EOL> from sklearn . feature_extraction import FeatureHasher <EOL> from sklearn . datasets . samples_generator import make_classification <EOL> from sklearn . utils . testing import assert_greater , assert_less , assert_equal , assert_almost_equal <EOL> from skll . config import _load_cv_folds <EOL> from skll . data import FeatureSet <EOL> from skll . learner import Learner <EOL> from skll . learner import _DEFAULT_PARAM_GRIDS <EOL> from skll . experiments import _load_featureset <EOL> from sklearn . cross_validation import StratifiedKFold <EOL> from utils import fill_in_config_paths_for_single_file <EOL> from skll . experiments import run_configuration <EOL> _ALL_MODELS = list ( _DEFAULT_PARAM_GRIDS . keys ( ) ) <EOL> _my_dir = abspath ( dirname ( __file__ ) ) <EOL> def setup ( ) : <EOL> """<STR_LIT>""" <EOL> train_dir = join ( _my_dir , '<STR_LIT:train>' ) <EOL> if not exists ( train_dir ) : <EOL> os . makedirs ( train_dir ) <EOL> output_dir = join ( _my_dir , '<STR_LIT>' ) <EOL> if not exists ( output_dir ) : <EOL> os . makedirs ( output_dir ) <EOL> def tearDown ( ) : <EOL> """<STR_LIT>""" <EOL> fold_file_path = join ( _my_dir , '<STR_LIT>' , '<STR_LIT>' ) <EOL> if exists ( fold_file_path ) : <EOL> os . unlink ( fold_file_path ) <EOL> train_dir = join ( _my_dir , '<STR_LIT:train>' ) <EOL> output_dir = join ( _my_dir , '<STR_LIT>' ) <EOL> config_dir = join ( _my_dir , '<STR_LIT>' ) <EOL> cfg_file = join ( config_dir , '<STR_LIT>' ) <EOL> os . unlink ( cfg_file ) <EOL> for output_file in ( glob ( join ( output_dir , <EOL> '<STR_LIT>' ) ) + <EOL> glob ( join ( output_dir , <EOL> '<STR_LIT>' ) ) ) : <EOL> os . unlink ( output_file ) <EOL> def make_cv_folds_data ( num_examples_per_fold = <NUM_LIT:100> , <EOL> num_folds = <NUM_LIT:3> , <EOL> use_feature_hashing = False ) : <EOL> """<STR_LIT>""" <EOL> num_total_examples = num_examples_per_fold * num_folds <EOL> X , _ = make_classification ( n_samples = num_total_examples , <EOL> n_features = <NUM_LIT:3> , n_informative = <NUM_LIT:3> , n_redundant = <NUM_LIT:0> , <EOL> n_classes = <NUM_LIT:2> , random_state = <NUM_LIT> ) <EOL> y = np . array ( [ <NUM_LIT:0> , <NUM_LIT:1> ] * int ( num_total_examples / <NUM_LIT:2> ) ) <EOL> foldgen = ( [ str ( i ) ] * num_examples_per_fold for i in range ( num_folds ) ) <EOL> folds = list ( itertools . chain ( * foldgen ) ) <EOL> feature_names = [ '<STR_LIT>' . format ( i ) for i in range ( <NUM_LIT:1> , <NUM_LIT:4> ) ] <EOL> features = [ ] <EOL> for row , classid , foldnum in zip ( X , y , folds ) : <EOL> string_feature_name = '<STR_LIT>' . format ( classid , foldnum ) <EOL> string_feature_value = <NUM_LIT:1> <EOL> feat_dict = dict ( zip ( feature_names , row ) ) <EOL> feat_dict . update ( { string_feature_name : string_feature_value } ) <EOL> features . append ( feat_dict ) <EOL> ids = [ '<STR_LIT>' . format ( num_examples_per_fold * k + i ) <EOL> for k in range ( num_folds ) for i in range ( num_examples_per_fold ) ] <EOL> vectorizer = FeatureHasher ( n_features = <NUM_LIT:4> ) if use_feature_hashing else None <EOL> cv_fs = FeatureSet ( '<STR_LIT>' , ids , features = features , labels = y , <EOL> vectorizer = vectorizer ) <EOL> custom_cv_folds = dict ( zip ( ids , folds ) ) <EOL> return ( cv_fs , custom_cv_folds ) <EOL> def test_specified_cv_folds ( ) : <EOL> """<STR_LIT>""" <EOL> for test_value , assert_func , grid_size , use_hashing , use_sampler in [ ( <NUM_LIT> , assert_less , <NUM_LIT:3> , False , False ) , <EOL> ( <NUM_LIT:0.1> , assert_greater , <NUM_LIT:10> , True , False ) , <EOL> ( <NUM_LIT> , assert_less , <NUM_LIT:3> , False , True ) , <EOL> ( <NUM_LIT> , assert_greater , <NUM_LIT:10> , True , True ) ] : <EOL> sampler = '<STR_LIT>' if use_sampler else None <EOL> learner = Learner ( '<STR_LIT>' , sampler = sampler ) <EOL> cv_fs , custom_cv_folds = make_cv_folds_data ( <EOL> use_feature_hashing = use_hashing ) <EOL> folds = custom_cv_folds if not use_hashing else <NUM_LIT:10> <EOL> cv_output = learner . cross_validate ( cv_fs , <EOL> cv_folds = folds , <EOL> grid_search = True ) <EOL> fold_test_scores = [ t [ - <NUM_LIT:1> ] for t in cv_output [ <NUM_LIT:0> ] ] <EOL> overall_score = np . mean ( fold_test_scores ) <EOL> assert_func ( overall_score , test_value ) <EOL> eq_ ( len ( fold_test_scores ) , grid_size ) <EOL> for fold_score in fold_test_scores : <EOL> assert_func ( fold_score , test_value ) <EOL> def test_load_cv_folds ( ) : <EOL> """<STR_LIT>""" <EOL> custom_cv_folds = make_cv_folds_data ( ) [ <NUM_LIT:1> ] <EOL> fold_file_path = join ( _my_dir , '<STR_LIT>' , '<STR_LIT>' ) <EOL> with open ( fold_file_path , '<STR_LIT:wb>' if PY2 else '<STR_LIT:w>' ) as foldf : <EOL> w = csv . writer ( foldf ) <EOL> w . writerow ( [ '<STR_LIT:id>' , '<STR_LIT>' ] ) <EOL> for example_id , fold_label in custom_cv_folds . items ( ) : <EOL> w . writerow ( [ example_id , fold_label ] ) <EOL> custom_cv_folds_loaded = _load_cv_folds ( fold_file_path ) <EOL> eq_ ( custom_cv_folds_loaded , custom_cv_folds ) <EOL> @ raises ( ValueError ) <EOL> def test_load_cv_folds_non_float_ids ( ) : <EOL> """<STR_LIT>""" <EOL> custom_cv_folds = make_cv_folds_data ( ) [ <NUM_LIT:1> ] <EOL> fold_file_path = join ( _my_dir , '<STR_LIT>' , '<STR_LIT>' ) <EOL> with open ( fold_file_path , '<STR_LIT:wb>' if PY2 else '<STR_LIT:w>' ) as foldf : <EOL> w = csv . writer ( foldf ) <EOL> w . writerow ( [ '<STR_LIT:id>' , '<STR_LIT>' ] ) <EOL> for example_id , fold_label in custom_cv_folds . items ( ) : <EOL> w . writerow ( [ example_id , fold_label ] ) <EOL> _load_cv_folds ( fold_file_path , ids_to_floats = True ) <EOL> def test_retrieve_cv_folds ( ) : <EOL> """<STR_LIT>""" <EOL> learner = Learner ( '<STR_LIT>' ) <EOL> num_folds = <NUM_LIT:5> <EOL> cv_fs , custom_cv_folds = make_cv_folds_data ( num_examples_per_fold = <NUM_LIT:2> , num_folds = num_folds ) <EOL> expected_fold_ids = { '<STR_LIT>' : '<STR_LIT:0>' , <EOL> '<STR_LIT>' : '<STR_LIT:4>' , <EOL> '<STR_LIT>' : '<STR_LIT:3>' , <EOL> '<STR_LIT>' : '<STR_LIT:1>' , <EOL> '<STR_LIT>' : '<STR_LIT:2>' , <EOL> '<STR_LIT>' : '<STR_LIT:2>' , <EOL> '<STR_LIT>' : '<STR_LIT:1>' , <EOL> '<STR_LIT>' : '<STR_LIT:0>' , <EOL> '<STR_LIT>' : '<STR_LIT:4>' , <EOL> '<STR_LIT>' : '<STR_LIT:3>' } <EOL> _ , _ , skll_fold_ids = learner . cross_validate ( cv_fs , <EOL> stratified = True , <EOL> cv_folds = num_folds , <EOL> grid_search = True , <EOL> shuffle = False , <EOL> save_cv_folds = True ) <EOL> assert_equal ( skll_fold_ids , expected_fold_ids ) <EOL> _ , _ , skll_fold_ids = learner . cross_validate ( cv_fs , <EOL> stratified = True , <EOL> cv_folds = custom_cv_folds , <EOL> grid_search = True , <EOL> shuffle = False , <EOL> save_cv_folds = True ) <EOL> assert_equal ( skll_fold_ids , custom_cv_folds ) <EOL> expected_fold_ids = { '<STR_LIT>' : '<STR_LIT:0>' , <EOL> '<STR_LIT>' : '<STR_LIT:0>' , <EOL> '<STR_LIT>' : '<STR_LIT:1>' , <EOL> '<STR_LIT>' : '<STR_LIT:1>' , <EOL> '<STR_LIT>' : '<STR_LIT:2>' , <EOL> '<STR_LIT>' : '<STR_LIT:2>' , <EOL> '<STR_LIT>' : '<STR_LIT:3>' , <EOL> '<STR_LIT>' : '<STR_LIT:3>' , <EOL> '<STR_LIT>' : '<STR_LIT:4>' , <EOL> '<STR_LIT>' : '<STR_LIT:4>' } <EOL> _ , _ , skll_fold_ids = learner . cross_validate ( cv_fs , <EOL> stratified = False , <EOL> cv_folds = num_folds , <EOL> grid_search = False , <EOL> shuffle = False , <EOL> save_cv_folds = True ) <EOL> assert_equal ( skll_fold_ids , custom_cv_folds ) <EOL> def test_cross_validate_task ( ) : <EOL> """<STR_LIT>""" <EOL> suffix = '<STR_LIT>' <EOL> train_path = join ( _my_dir , '<STR_LIT:train>' , '<STR_LIT>' . format ( suffix ) ) <EOL> config_path = fill_in_config_paths_for_single_file ( join ( _my_dir , "<STR_LIT>" , <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> train_path , <EOL> None ) <EOL> run_configuration ( config_path , quiet = True ) <EOL> with open ( join ( _my_dir , '<STR_LIT>' , '<STR_LIT>' + <EOL> '<STR_LIT>' ) ) as f : <EOL> result_dict = json . load ( f ) [ <NUM_LIT:10> ] <EOL> assert_almost_equal ( result_dict [ '<STR_LIT>' ] , <NUM_LIT> ) <EOL> expected_skll_ids = { } <EOL> examples = _load_featureset ( train_path , '<STR_LIT>' , suffix , quiet = True ) <EOL> kfold = StratifiedKFold ( examples . labels , n_folds = <NUM_LIT:10> ) <EOL> for fold_num , ( _ , test_indices ) in enumerate ( kfold ) : <EOL> for index in test_indices : <EOL> expected_skll_ids [ examples . ids [ index ] ] = fold_num <EOL> skll_fold_ids = { } <EOL> with open ( join ( _my_dir , '<STR_LIT>' , '<STR_LIT>' ) ) as f : <EOL> reader = csv . DictReader ( f ) <EOL> for row in reader : <EOL> skll_fold_ids [ row [ '<STR_LIT:id>' ] ] = row [ '<STR_LIT>' ] <EOL> skll_fold_ids_str = '<STR_LIT>' . join ( '<STR_LIT>' . format ( key , val ) for key , val in sorted ( skll_fold_ids . items ( ) ) ) <EOL> expected_skll_ids_str = '<STR_LIT>' . join ( '<STR_LIT>' . format ( key , val ) for key , val in sorted ( expected_skll_ids . items ( ) ) ) <EOL> assert_equal ( skll_fold_ids_str , expected_skll_ids_str ) </s>
