<s> """<STR_LIT>""" <EOL> from __future__ import ( absolute_import , division , print_function , <EOL> unicode_literals ) <EOL> import re <EOL> from os . path import abspath , dirname , join <EOL> import numpy as np <EOL> from numpy . random import RandomState <EOL> from sklearn . datasets . samples_generator import ( make_classification , <EOL> make_regression ) <EOL> from sklearn . feature_extraction import FeatureHasher <EOL> from skll . data import FeatureSet <EOL> from skll . config import _setup_config_parser <EOL> _my_dir = abspath ( dirname ( __file__ ) ) <EOL> def fill_in_config_paths ( config_template_path ) : <EOL> """<STR_LIT>""" <EOL> train_dir = join ( _my_dir , '<STR_LIT:train>' ) <EOL> test_dir = join ( _my_dir , '<STR_LIT:test>' ) <EOL> output_dir = join ( _my_dir , '<STR_LIT>' ) <EOL> config = _setup_config_parser ( config_template_path , validate = False ) <EOL> task = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , train_dir ) <EOL> to_fill_in = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> if task != '<STR_LIT>' : <EOL> to_fill_in . append ( '<STR_LIT>' ) <EOL> if task == '<STR_LIT>' or task == '<STR_LIT>' : <EOL> to_fill_in . append ( '<STR_LIT>' ) <EOL> for d in to_fill_in : <EOL> config . set ( "<STR_LIT>" , d , join ( output_dir ) ) <EOL> if task == '<STR_LIT>' : <EOL> cv_folds_file = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> if cv_folds_file : <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , <EOL> join ( train_dir , cv_folds_file ) ) <EOL> if task == '<STR_LIT>' or task == '<STR_LIT>' : <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , test_dir ) <EOL> custom_learner_path = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> custom_learner_abs_path = join ( _my_dir , custom_learner_path ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , custom_learner_abs_path ) <EOL> config_prefix = re . search ( r'<STR_LIT>' , <EOL> config_template_path ) . groups ( ) [ <NUM_LIT:0> ] <EOL> new_config_path = '<STR_LIT>' . format ( config_prefix ) <EOL> with open ( new_config_path , '<STR_LIT:w>' ) as new_config_file : <EOL> config . write ( new_config_file ) <EOL> return new_config_path <EOL> def fill_in_config_paths_for_single_file ( config_template_path , train_file , <EOL> test_file , train_directory = '<STR_LIT>' , <EOL> test_directory = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> train_dir = join ( _my_dir , '<STR_LIT:train>' ) <EOL> test_dir = join ( _my_dir , '<STR_LIT:test>' ) <EOL> output_dir = join ( _my_dir , '<STR_LIT>' ) <EOL> config = _setup_config_parser ( config_template_path , validate = False ) <EOL> task = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , join ( train_dir , train_file ) ) <EOL> if task == '<STR_LIT>' or task == '<STR_LIT>' : <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , join ( test_dir , test_file ) ) <EOL> if train_directory : <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , join ( train_dir , train_directory ) ) <EOL> if test_directory : <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , join ( test_dir , test_directory ) ) <EOL> to_fill_in = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> if task != '<STR_LIT>' : <EOL> to_fill_in . append ( '<STR_LIT>' ) <EOL> if task == '<STR_LIT>' or task == '<STR_LIT>' : <EOL> to_fill_in . append ( '<STR_LIT>' ) <EOL> for d in to_fill_in : <EOL> config . set ( "<STR_LIT>" , d , join ( output_dir ) ) <EOL> if task == '<STR_LIT>' : <EOL> cv_folds_file = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> if cv_folds_file : <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , <EOL> join ( train_dir , cv_folds_file ) ) <EOL> config_prefix = re . search ( r'<STR_LIT>' , <EOL> config_template_path ) . groups ( ) [ <NUM_LIT:0> ] <EOL> new_config_path = '<STR_LIT>' . format ( config_prefix ) <EOL> with open ( new_config_path , '<STR_LIT:w>' ) as new_config_file : <EOL> config . write ( new_config_file ) <EOL> return new_config_path <EOL> def fill_in_config_options ( config_template_path , <EOL> values_to_fill_dict , <EOL> sub_prefix ) : <EOL> """<STR_LIT>""" <EOL> config = _setup_config_parser ( config_template_path , validate = False ) <EOL> to_fill_in = { '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] } <EOL> for section in to_fill_in : <EOL> for param_name in to_fill_in [ section ] : <EOL> if param_name in values_to_fill_dict : <EOL> config . set ( section , param_name , <EOL> values_to_fill_dict [ param_name ] ) <EOL> config_prefix = re . search ( r'<STR_LIT>' , <EOL> config_template_path ) . groups ( ) [ <NUM_LIT:0> ] <EOL> new_config_path = '<STR_LIT>' . format ( config_prefix , sub_prefix ) <EOL> with open ( new_config_path , '<STR_LIT:w>' ) as new_config_file : <EOL> config . write ( new_config_file ) <EOL> return new_config_path <EOL> def fill_in_config_paths_for_fancy_output ( config_template_path ) : <EOL> """<STR_LIT>""" <EOL> train_dir = join ( _my_dir , '<STR_LIT:train>' ) <EOL> test_dir = join ( _my_dir , '<STR_LIT:test>' ) <EOL> output_dir = join ( _my_dir , '<STR_LIT>' ) <EOL> config = _setup_config_parser ( config_template_path , validate = False ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , join ( train_dir , "<STR_LIT>" ) ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , join ( test_dir , <EOL> "<STR_LIT>" ) ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , output_dir ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , output_dir ) <EOL> config . set ( "<STR_LIT>" , "<STR_LIT>" , output_dir ) <EOL> config_prefix = re . search ( r'<STR_LIT>' , <EOL> config_template_path ) . groups ( ) [ <NUM_LIT:0> ] <EOL> new_config_path = '<STR_LIT>' . format ( config_prefix ) <EOL> with open ( new_config_path , '<STR_LIT:w>' ) as new_config_file : <EOL> config . write ( new_config_file ) <EOL> return new_config_path <EOL> def make_classification_data ( num_examples = <NUM_LIT:100> , train_test_ratio = <NUM_LIT:0.5> , <EOL> num_features = <NUM_LIT:10> , use_feature_hashing = False , <EOL> feature_bins = <NUM_LIT:4> , num_labels = <NUM_LIT:2> , <EOL> empty_labels = False , feature_prefix = '<STR_LIT:f>' , <EOL> class_weights = None , non_negative = False , <EOL> one_string_feature = False , num_string_values = <NUM_LIT:4> , <EOL> random_state = <NUM_LIT> ) : <EOL> num_numeric_features = ( num_features - <NUM_LIT:1> if one_string_feature else <EOL> num_features ) <EOL> X , y = make_classification ( n_samples = num_examples , <EOL> n_features = num_numeric_features , <EOL> n_informative = num_numeric_features , <EOL> n_redundant = <NUM_LIT:0> , n_classes = num_labels , <EOL> weights = class_weights , <EOL> random_state = random_state ) <EOL> if non_negative : <EOL> X = abs ( X ) <EOL> ids = [ '<STR_LIT>' . format ( n ) for n in range ( <NUM_LIT:1> , num_examples + <NUM_LIT:1> ) ] <EOL> if one_string_feature : <EOL> prng = RandomState ( random_state ) <EOL> random_indices = prng . random_integers ( <NUM_LIT:0> , num_string_values - <NUM_LIT:1> , <EOL> num_examples ) <EOL> possible_values = [ chr ( x ) for x in range ( <NUM_LIT> , <NUM_LIT> + num_string_values ) ] <EOL> string_feature_values = [ possible_values [ i ] for i in random_indices ] <EOL> string_feature_column = np . array ( string_feature_values , <EOL> dtype = object ) . reshape ( <NUM_LIT:100> , <NUM_LIT:1> ) <EOL> X = np . append ( X , string_feature_column , <NUM_LIT:1> ) <EOL> feature_names = [ '<STR_LIT>' . format ( feature_prefix , n ) for n in <EOL> range ( <NUM_LIT:1> , num_features + <NUM_LIT:1> ) ] <EOL> features = [ dict ( zip ( feature_names , row ) ) for row in X ] <EOL> num_train_examples = int ( round ( train_test_ratio * num_examples ) ) <EOL> train_features , test_features = ( features [ : num_train_examples ] , <EOL> features [ num_train_examples : ] ) <EOL> train_y , test_y = y [ : num_train_examples ] , y [ num_train_examples : ] <EOL> train_ids , test_ids = ids [ : num_train_examples ] , ids [ num_train_examples : ] <EOL> train_labels = None if empty_labels else train_y <EOL> test_labels = None if empty_labels else test_y <EOL> vectorizer = ( FeatureHasher ( n_features = feature_bins ) <EOL> if use_feature_hashing else None ) <EOL> train_fs = FeatureSet ( '<STR_LIT>' , train_ids , <EOL> labels = train_labels , features = train_features , <EOL> vectorizer = vectorizer ) <EOL> if train_test_ratio < <NUM_LIT:1.0> : <EOL> test_fs = FeatureSet ( '<STR_LIT>' , test_ids , <EOL> labels = test_labels , features = test_features , <EOL> vectorizer = vectorizer ) <EOL> else : <EOL> test_fs = None <EOL> return ( train_fs , test_fs ) <EOL> def make_regression_data ( num_examples = <NUM_LIT:100> , train_test_ratio = <NUM_LIT:0.5> , <EOL> num_features = <NUM_LIT:2> , sd_noise = <NUM_LIT:1.0> , <EOL> use_feature_hashing = False , <EOL> feature_bins = <NUM_LIT:4> , <EOL> start_feature_num = <NUM_LIT:1> , <EOL> random_state = <NUM_LIT> ) : <EOL> X , y , weights = make_regression ( n_samples = num_examples , <EOL> n_features = num_features , <EOL> noise = sd_noise , random_state = random_state , <EOL> coef = True ) <EOL> ids = [ '<STR_LIT>' . format ( n ) for n in range ( <NUM_LIT:1> , num_examples + <NUM_LIT:1> ) ] <EOL> feature_names = [ '<STR_LIT>' . format ( n ) for n <EOL> in range ( start_feature_num , <EOL> start_feature_num + num_features ) ] <EOL> features = [ dict ( zip ( feature_names , row ) ) for row in X ] <EOL> weightdict = dict ( zip ( feature_names , weights ) ) <EOL> num_train_examples = int ( round ( train_test_ratio * num_examples ) ) <EOL> train_features , test_features = ( features [ : num_train_examples ] , <EOL> features [ num_train_examples : ] ) <EOL> train_y , test_y = y [ : num_train_examples ] , y [ num_train_examples : ] <EOL> train_ids , test_ids = ids [ : num_train_examples ] , ids [ num_train_examples : ] <EOL> vectorizer = ( FeatureHasher ( n_features = feature_bins ) if <EOL> use_feature_hashing else None ) <EOL> train_fs = FeatureSet ( '<STR_LIT>' , train_ids , <EOL> labels = train_y , features = train_features , <EOL> vectorizer = vectorizer ) <EOL> test_fs = FeatureSet ( '<STR_LIT>' , test_ids , <EOL> labels = test_y , features = test_features , <EOL> vectorizer = vectorizer ) <EOL> return ( train_fs , test_fs , weightdict ) <EOL> def make_sparse_data ( use_feature_hashing = False ) : <EOL> """<STR_LIT>""" <EOL> X , y = make_classification ( n_samples = <NUM_LIT> , n_features = <NUM_LIT:3> , <EOL> n_informative = <NUM_LIT:3> , n_redundant = <NUM_LIT:0> , <EOL> n_classes = <NUM_LIT:2> , random_state = <NUM_LIT> ) <EOL> X = np . abs ( X ) <EOL> X [ np . where ( X == <NUM_LIT:0> ) ] += <NUM_LIT:1> <EOL> ids = [ '<STR_LIT>' . format ( n ) for n in range ( <NUM_LIT:1> , <NUM_LIT> ) ] <EOL> feature_names = [ '<STR_LIT>' . format ( n ) for n in range ( <NUM_LIT:1> , <NUM_LIT:6> ) ] <EOL> features = [ ] <EOL> for row in X : <EOL> row = [ <NUM_LIT:0> ] + row . tolist ( ) + [ <NUM_LIT:0> ] <EOL> features . append ( dict ( zip ( feature_names , row ) ) ) <EOL> vectorizer = FeatureHasher ( n_features = <NUM_LIT:4> ) if use_feature_hashing else None <EOL> train_fs = FeatureSet ( '<STR_LIT>' , ids , <EOL> features = features , labels = y , <EOL> vectorizer = vectorizer ) <EOL> X , y = make_classification ( n_samples = <NUM_LIT:100> , n_features = <NUM_LIT:4> , <EOL> n_informative = <NUM_LIT:4> , n_redundant = <NUM_LIT:0> , <EOL> n_classes = <NUM_LIT:2> , random_state = <NUM_LIT> ) <EOL> X = np . abs ( X ) <EOL> X [ np . where ( X == <NUM_LIT:0> ) ] += <NUM_LIT:1> <EOL> ids = [ '<STR_LIT>' . format ( n ) for n in range ( <NUM_LIT:1> , <NUM_LIT> ) ] <EOL> feature_names = [ '<STR_LIT>' . format ( n ) for n in range ( <NUM_LIT:1> , <NUM_LIT:6> ) ] <EOL> features = [ ] <EOL> for row in X : <EOL> row = row . tolist ( ) <EOL> row = row [ : <NUM_LIT:3> ] + [ <NUM_LIT:0> ] + row [ <NUM_LIT:3> : ] <EOL> features . append ( dict ( zip ( feature_names , row ) ) ) <EOL> test_fs = FeatureSet ( '<STR_LIT>' , ids , <EOL> features = features , labels = y , <EOL> vectorizer = vectorizer ) <EOL> return train_fs , test_fs </s>
