<s> """<STR_LIT>""" <EOL> __author__ = '<STR_LIT>' <EOL> __email__ = '<STR_LIT>' <EOL> __date__ = '<STR_LIT>' <EOL> import utils <EOL> log = utils . start_logging ( __name__ ) <EOL> from sklearn . feature_extraction import text , DictVectorizer <EOL> from sklearn import preprocessing <EOL> from scipy . sparse import coo_matrix , hstack , vstack <EOL> import numpy as np <EOL> import pandas as pd <EOL> from datetime import datetime <EOL> from sklearn import neighbors <EOL> def add ( df ) : <EOL> hours ( df ) <EOL> age ( df ) <EOL> city ( df ) <EOL> lat_long ( df ) <EOL> dayofweek ( df ) <EOL> description_length ( df ) <EOL> description_length_log ( df ) <EOL> description_fg ( df ) <EOL> tagtype_fg ( df ) <EOL> weekend_fg ( df ) <EOL> nbr_longlat ( df ) <EOL> def hours ( df ) : <EOL> df [ '<STR_LIT>' ] = [ str ( x . hour ) for x in df [ '<STR_LIT>' ] ] <EOL> df [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> for idx in df . index : <EOL> if df [ '<STR_LIT>' ] [ idx ] . hour in [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> ] : <EOL> df [ '<STR_LIT>' ] [ idx ] = '<STR_LIT>' <EOL> if df [ '<STR_LIT>' ] [ idx ] . hour in [ <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> , <NUM_LIT:10> , <NUM_LIT:11> ] : <EOL> df [ '<STR_LIT>' ] [ idx ] = '<STR_LIT>' <EOL> if df [ '<STR_LIT>' ] [ idx ] . hour in [ <NUM_LIT:12> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT:15> , <NUM_LIT:16> , <NUM_LIT> ] : <EOL> df [ '<STR_LIT>' ] [ idx ] = '<STR_LIT>' <EOL> def age ( df ) : <EOL> df [ '<STR_LIT>' ] = [ ( datetime . now ( ) . date ( ) - x ) . days for x in df [ '<STR_LIT>' ] ] <EOL> def city ( df ) : <EOL> df [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> for idx in df . index : <EOL> if df [ '<STR_LIT>' ] [ idx ] < - <NUM_LIT> and df [ '<STR_LIT>' ] [ idx ] > - <NUM_LIT> : <EOL> df [ '<STR_LIT>' ] [ idx ] = '<STR_LIT>' <EOL> if df [ '<STR_LIT>' ] [ idx ] < - <NUM_LIT> and df [ '<STR_LIT>' ] [ idx ] > - <NUM_LIT> : <EOL> df [ '<STR_LIT>' ] [ idx ] = '<STR_LIT>' <EOL> if df [ '<STR_LIT>' ] [ idx ] < - <NUM_LIT> and df [ '<STR_LIT>' ] [ idx ] > - <NUM_LIT> : <EOL> df [ '<STR_LIT>' ] [ idx ] = '<STR_LIT>' <EOL> def nbr_longlat ( df ) : <EOL> df [ '<STR_LIT>' ] = df [ '<STR_LIT>' ] <EOL> for idx in df . index : <EOL> if df [ '<STR_LIT>' ] [ idx ] in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> df [ '<STR_LIT>' ] [ idx ] = df [ '<STR_LIT>' ] [ idx ] <EOL> def lat_long ( df ) : <EOL> df [ '<STR_LIT>' ] = [ str ( round ( x , <NUM_LIT:2> ) ) for x in df . longitude ] <EOL> df [ '<STR_LIT>' ] = [ str ( round ( x , <NUM_LIT:2> ) ) for x in df . latitude ] <EOL> df [ '<STR_LIT>' ] = df [ '<STR_LIT>' ] + df [ '<STR_LIT>' ] <EOL> def dayofweek ( df ) : <EOL> df [ '<STR_LIT>' ] = [ str ( x . weekday ( ) ) for x in df [ '<STR_LIT>' ] ] <EOL> def weekend_fg ( df ) : <EOL> df [ '<STR_LIT>' ] = [ <NUM_LIT:0> if x < <NUM_LIT:4> else <NUM_LIT:1> for x in df [ '<STR_LIT>' ] ] <EOL> def holidays ( df , holidays_list ) : <EOL> df [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> df [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> for idx in df . index : <EOL> for holiday in holidays_list : <EOL> if holiday . lower ( ) in df . text_all [ idx ] . lower ( ) : <EOL> df [ '<STR_LIT>' ] [ idx ] = <NUM_LIT:1> <EOL> df [ '<STR_LIT>' ] [ idx ] = holiday <EOL> def description_length ( df ) : <EOL> df [ '<STR_LIT>' ] = [ - <NUM_LIT> if x == '<STR_LIT>' else len ( x . split ( ) ) for x in df . description ] <EOL> def description_length_log ( df ) : <EOL> df [ '<STR_LIT>' ] = [ <NUM_LIT:0> if x == '<STR_LIT>' else len ( x . split ( ) ) for x in df . description ] <EOL> df [ '<STR_LIT>' ] = np . log ( df . description_length_log + <NUM_LIT:1> ) <EOL> def description_fg ( df ) : <EOL> df [ '<STR_LIT>' ] = [ <NUM_LIT:0> if x == '<STR_LIT>' else <NUM_LIT:1> for x in df . description ] <EOL> def tagtype_fg ( df ) : <EOL> df [ '<STR_LIT>' ] = [ <NUM_LIT:0> if x == '<STR_LIT>' else <NUM_LIT:1> for x in df . tag_type ] <EOL> def vectors ( dfTrn , dfTest , model_features ) : <EOL> for feature in model_features : <EOL> if feature == '<STR_LIT>' : <EOL> summary_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . summary , dfTest . summary ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = summary_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . summary ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = summary_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . summary ] ) <EOL> if feature == '<STR_LIT>' : <EOL> count_vec = text . CountVectorizer ( min_df = <NUM_LIT:3> , max_df = <NUM_LIT> , strip_accents = '<STR_LIT>' , binary = True ) <EOL> count_vec . fit ( np . append ( dfTrn . summary . values , dfTest . summary . values ) ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = count_vec . transform ( dfTrn . summary . values ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = count_vec . transform ( dfTest . summary . values ) <EOL> if feature == '<STR_LIT>' : <EOL> tfidf_vec = text . TfidfVectorizer ( min_df = <NUM_LIT:6> , max_features = None , strip_accents = '<STR_LIT>' , analyzer = '<STR_LIT>' , token_pattern = r'<STR_LIT>' , ngram_range = ( <NUM_LIT:1> , <NUM_LIT:2> ) , use_idf = <NUM_LIT:1> , smooth_idf = <NUM_LIT:1> , sublinear_tf = <NUM_LIT:1> ) <EOL> tfidf_vec . fit ( np . append ( dfTrn . summary . values , dfTest . summary . values ) ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = tfidf_vec . transform ( ( dfTrn . summary . values ) ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = tfidf_vec . transform ( ( dfTest . summary . values ) ) <EOL> if feature == '<STR_LIT>' : <EOL> tfidf_vec = text . TfidfVectorizer ( min_df = <NUM_LIT:3> , max_features = None , strip_accents = '<STR_LIT>' , analyzer = '<STR_LIT>' , token_pattern = r'<STR_LIT>' , ngram_range = ( <NUM_LIT:2> , <NUM_LIT:30> ) , use_idf = <NUM_LIT:1> , smooth_idf = <NUM_LIT:1> , sublinear_tf = <NUM_LIT:1> ) <EOL> tfidf_vec . fit ( np . append ( dfTrn . summary . values , dfTest . summary . values ) ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = tfidf_vec . transform ( ( dfTrn . summary . values ) ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = tfidf_vec . transform ( ( dfTest . summary . values ) ) <EOL> if feature == '<STR_LIT>' : <EOL> tfidf_vec = text . TfidfVectorizer ( min_df = <NUM_LIT:6> , max_features = None , strip_accents = '<STR_LIT>' , analyzer = '<STR_LIT>' , token_pattern = r'<STR_LIT>' , ngram_range = ( <NUM_LIT:1> , <NUM_LIT:2> ) , use_idf = <NUM_LIT:1> , smooth_idf = <NUM_LIT:1> , sublinear_tf = <NUM_LIT:1> ) <EOL> tfidf_vec . fit ( np . append ( np . append ( dfTrn . summary . values , dfTrn . description . values ) , np . append ( dfTest . summary . values , dfTest . description . values ) ) ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = tfidf_vec . transform ( ( dfTrn . summary . values ) ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = tfidf_vec . transform ( ( dfTest . summary . values ) ) <EOL> if feature == '<STR_LIT>' : <EOL> tfidf_vec = text . TfidfVectorizer ( min_df = <NUM_LIT:3> , max_features = None , strip_accents = '<STR_LIT>' , analyzer = '<STR_LIT>' , token_pattern = r'<STR_LIT>' , ngram_range = ( <NUM_LIT:2> , <NUM_LIT:30> ) , use_idf = <NUM_LIT:1> , smooth_idf = <NUM_LIT:1> , sublinear_tf = <NUM_LIT:1> ) <EOL> tfidf_vec . fit ( np . append ( np . append ( dfTrn . summary . values , dfTrn . description . values ) , np . append ( dfTest . summary . values , dfTest . description . values ) ) ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = tfidf_vec . transform ( ( dfTrn . summary . values ) ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = tfidf_vec . transform ( ( dfTest . summary . values ) ) <EOL> if feature == '<STR_LIT>' : <EOL> city_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . city , dfTest . city ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = city_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . city ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = city_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . city ] ) <EOL> if feature == '<STR_LIT>' : <EOL> tagtype_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . tag_type , dfTest . tag_type ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = tagtype_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . tag_type ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = tagtype_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . tag_type ] ) <EOL> if feature == '<STR_LIT:source>' : <EOL> source_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . source , dfTest . source ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = source_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . source ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = source_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . source ] ) <EOL> if feature == '<STR_LIT>' : <EOL> longlat_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . long_lat_rnd2 , dfTest . long_lat_rnd2 ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = longlat_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . long_lat_rnd2 ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = longlat_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . long_lat_rnd2 ] ) <EOL> if feature == '<STR_LIT>' : <EOL> neighborhood_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . neighborhood , dfTest . neighborhood ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = neighborhood_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . neighborhood ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = neighborhood_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . neighborhood ] ) <EOL> if feature == '<STR_LIT>' : <EOL> street_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . street , dfTest . street ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = street_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . street ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = street_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . neighborhood ] ) <EOL> if feature == '<STR_LIT>' : <EOL> zipcode_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . zipcode , dfTest . zipcode ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = zipcode_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . zipcode ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = zipcode_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . zipcode ] ) <EOL> if feature == '<STR_LIT>' : <EOL> hrsrange_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . created_time_range , dfTest . created_time_range ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = hrsrange_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . created_time_range ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = hrsrange_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . created_time_range ] ) <EOL> if feature == '<STR_LIT>' : <EOL> dayofweek_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . dayofweek , dfTest . dayofweek ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = dayofweek_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . dayofweek ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = dayofweek_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . dayofweek ] ) <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = dfTrn . ix [ : , [ '<STR_LIT>' ] ] . as_matrix ( ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = dfTest . ix [ : , [ '<STR_LIT>' ] ] . as_matrix ( ) <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = dfTrn . ix [ : , [ '<STR_LIT>' ] ] . as_matrix ( ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = dfTest . ix [ : , [ '<STR_LIT>' ] ] . as_matrix ( ) <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = dfTrn . ix [ : , [ '<STR_LIT>' ] ] . as_matrix ( ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = dfTest . ix [ : , [ '<STR_LIT>' ] ] . as_matrix ( ) <EOL> if feature == '<STR_LIT>' : <EOL> hrs_vec = DictVectorizer ( ) . fit ( [ { '<STR_LIT>' : value } for value in np . append ( dfTrn . created_time_hrs , dfTest . created_time_hrs ) ] ) <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = source_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTrn . created_time_hrs ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = source_vec . transform ( [ { '<STR_LIT>' : value } for value in dfTest . created_time_hrs ] ) <EOL> def numerical ( dfTrn , dfTest , model_features ) : <EOL> for feature in model_features : <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = standardize ( dfTrn , [ feature ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = standardize ( dfTest , [ feature ] ) <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = standardize ( dfTrn , [ feature ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = standardize ( dfTest , [ feature ] ) <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = standardize ( dfTrn , [ feature ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = standardize ( dfTest , [ feature ] ) <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = standardize ( dfTrn , [ feature ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = standardize ( dfTest , [ feature ] ) <EOL> if feature == '<STR_LIT>' : <EOL> model_features [ feature ] [ <NUM_LIT:0> ] = standardize ( dfTrn , [ feature ] ) <EOL> model_features [ feature ] [ <NUM_LIT:1> ] = standardize ( dfTest , [ feature ] ) <EOL> def sub_feature ( df , feature1 , feature2 , values ) : <EOL> """<STR_LIT>""" <EOL> for idx in df . index : <EOL> if df [ feature2 ] [ idx ] in values : <EOL> df [ feature2 ] [ idx ] = str ( df [ feature1 ] [ idx ] ) <EOL> def knn_thresholding ( df , column , threshold = <NUM_LIT:15> , k = <NUM_LIT:3> ) : <EOL> """<STR_LIT>""" <EOL> def ids_centers_sizes ( data ) : <EOL> dat = np . array ( [ ( i , data . latitude [ data [ column ] == i ] . mean ( ) , <EOL> data . longitude [ data [ column ] == i ] . mean ( ) , <EOL> ( data [ column ] == i ) . sum ( ) ) <EOL> for i in set ( list ( data [ column ] ) ) ] ) <EOL> return dat [ : , <NUM_LIT:0> ] , dat [ : , <NUM_LIT:1> : - <NUM_LIT:1> ] . astype ( float ) , dat [ : , - <NUM_LIT:1> ] . astype ( int ) <EOL> knn = neighbors . NearestNeighbors ( n_neighbors = k ) <EOL> while True : <EOL> ids , centers , sizes = ids_centers_sizes ( df ) <EOL> asrt = np . argsort ( sizes ) <EOL> if sizes [ asrt [ <NUM_LIT:0> ] ] >= threshold : <EOL> break <EOL> cids = np . copy ( ids ) <EOL> knn . fit ( centers ) <EOL> for i in asrt : <EOL> if sizes [ i ] < threshold : <EOL> nearest = knn . kneighbors ( centers [ i ] ) [ <NUM_LIT:1> ] . flatten ( ) <EOL> nearest = nearest [ nearest != i ] <EOL> sel = nearest [ np . argmin ( sizes [ nearest ] ) ] <EOL> total_size = sizes [ sel ] + sizes [ i ] <EOL> df [ column ] [ df [ column ] == cids [ i ] ] = cids [ sel ] <EOL> cids [ cids == i ] = cids [ sel ] <EOL> sizes [ i ] = total_size <EOL> sizes [ sel ] = total_size <EOL> return df <EOL> def standardize ( df , features ) : <EOL> scaler = preprocessing . StandardScaler ( ) <EOL> if features == '<STR_LIT:all>' : <EOL> mtx = scaler . fit_transform ( df . as_matrix ( ) ) <EOL> else : <EOL> mtx = scaler . fit_transform ( df . ix [ : , features ] . as_matrix ( ) ) <EOL> return mtx </s>
