<s> from csv import DictReader <EOL> from datetime import datetime , timedelta <EOL> from collections import defaultdict <EOL> import cPickle as pickle <EOL> from math import exp , log , sqrt <EOL> import random , gc <EOL> from util import read_dump , write_dump , cache , read_tsv , convert_ts , data , next_row , get_category <EOL> import argparse , ast , re , json <EOL> def filter_row ( row , data_type , sr ) : <EOL> object_type = int ( row [ "<STR_LIT>" ] ) <EOL> if object_type != <NUM_LIT:3> : <EOL> return False <EOL> y = int ( row . get ( "<STR_LIT>" , <NUM_LIT:0> ) ) <EOL> if data_type == <NUM_LIT:0> and y == <NUM_LIT:0> and random . random ( ) > sr : <EOL> return False <EOL> return True <EOL> def calc_ctr ( x , y ) : <EOL> avg_ctr = <NUM_LIT> <EOL> return int ( round ( ( x + avg_ctr * <NUM_LIT:10> ) * <NUM_LIT> / ( y + <NUM_LIT:10> ) ) ) <EOL> def log_trans ( x ) : <EOL> if args . log > <NUM_LIT:0> : <EOL> return int ( round ( log ( x + <NUM_LIT:1> ) ) ) <EOL> return x <EOL> def get_user_info ( ) : <EOL> user_info_map = { } <EOL> for t , row in read_tsv ( "<STR_LIT>" ) : <EOL> for k in row : <EOL> row [ k ] = int ( row [ k ] ) <EOL> uid = row [ "<STR_LIT>" ] <EOL> del row [ "<STR_LIT>" ] <EOL> user_info_map [ uid ] = row <EOL> return user_info_map <EOL> def trans_ad_info ( ad_info ) : <EOL> if int ( ad_info [ "<STR_LIT>" ] ) == <NUM_LIT:0> : <EOL> return None <EOL> trans_keys = [ <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] <EOL> del_keys = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , ] <EOL> for k in del_keys : <EOL> if k in ad_info : <EOL> del ad_info [ k ] <EOL> for key in trans_keys : <EOL> val = ad_info [ key ] <EOL> if key == "<STR_LIT>" : <EOL> if val == "<STR_LIT>" : <EOL> pass <EOL> else : <EOL> ad_info [ key ] = float ( ad_info [ key ] ) <EOL> elif key == "<STR_LIT>" : <EOL> params = ad_info [ key ] <EOL> params = ast . literal_eval ( params ) if params else { } <EOL> for par_key in params : <EOL> params [ par_key ] = unicode ( params [ par_key ] , "<STR_LIT:utf-8>" ) <EOL> val = tuple ( [ hash_val ( <NUM_LIT:0> , ( k , v ) ) for k , v in params . items ( ) ] ) <EOL> if len ( val ) == <NUM_LIT:0> : <EOL> val = ( - <NUM_LIT:1> , ) <EOL> ad_info [ key ] = val <EOL> elif key == "<STR_LIT>" : <EOL> if not isinstance ( ad_info [ key ] , unicode ) : <EOL> ad_info [ key ] = unicode ( ad_info [ key ] , "<STR_LIT:utf-8>" ) <EOL> else : <EOL> if val == "<STR_LIT>" : <EOL> val = - <NUM_LIT:1> <EOL> ad_info [ key ] = int ( val ) <EOL> return ad_info <EOL> ad_info_list = [ ] <EOL> ad_info_iter = read_tsv ( "<STR_LIT>" ) <EOL> def get_ad_info ( aid ) : <EOL> while aid - <NUM_LIT:1> >= len ( ad_info_list ) : <EOL> t , row = next ( ad_info_iter , ( None , None ) ) <EOL> if row is None : <EOL> break <EOL> ad_info_list . append ( trans_ad_info ( row ) ) <EOL> return ad_info_list [ aid - <NUM_LIT:1> ] <EOL> se_params_iter = read_tsv ( "<STR_LIT>" , delimiter = "<STR_LIT:U+002C>" ) <EOL> se_param_list = [ None ] <EOL> def get_se_param ( sid ) : <EOL> while se_param_list [ <NUM_LIT:0> ] is None or se_param_list [ <NUM_LIT:0> ] [ "<STR_LIT>" ] < sid : <EOL> t , se_param = next ( se_params_iter , ( None , None ) ) <EOL> se_param [ "<STR_LIT>" ] = int ( se_param [ "<STR_LIT>" ] ) <EOL> params = json . loads ( se_param [ "<STR_LIT>" ] ) <EOL> se_param [ "<STR_LIT>" ] = [ hash_val ( <NUM_LIT:0> , ( int ( k ) , v ) ) for ( k , v ) in params . items ( ) ] <EOL> se_param_list [ <NUM_LIT:0> ] = se_param <EOL> params = [ - <NUM_LIT:1> , ] if se_param_list [ <NUM_LIT:0> ] [ "<STR_LIT>" ] != sid else se_param_list [ <NUM_LIT:0> ] [ "<STR_LIT>" ] <EOL> return params <EOL> ad_price_list = [ ] <EOL> ad_price_iter = read_tsv ( "<STR_LIT>" , delimiter = "<STR_LIT:U+0020>" ) <EOL> def get_ad_price ( aid ) : <EOL> while aid - <NUM_LIT:1> >= len ( ad_price_list ) : <EOL> t , row = next ( ad_price_iter , ( None , None ) ) <EOL> if row is None : <EOL> break <EOL> price = row [ "<STR_LIT>" ] <EOL> price = float ( price ) if price else "<STR_LIT>" <EOL> ad_price_list . append ( price ) <EOL> return ad_price_list [ aid - <NUM_LIT:1> ] <EOL> def get_features ( sinfo , rows , test = False ) : <EOL> feature_map = defaultdict ( list ) <EOL> sid = sinfo [ "<STR_LIT>" ] <EOL> sinfo [ "<STR_LIT>" ] = get_se_param ( sid ) <EOL> user_cnt_row = next ( user_cnt_iter , ( None , None ) ) [ <NUM_LIT:1> ] <EOL> while int ( user_cnt_row [ "<STR_LIT>" ] ) != sid : <EOL> user_cnt_row = next ( user_cnt_iter , ( None , None ) ) [ <NUM_LIT:1> ] <EOL> user_aid_cnt_rows = next ( user_aid_cnt_iter , ( None , None ) ) [ <NUM_LIT:1> ] <EOL> while int ( user_aid_cnt_rows [ <NUM_LIT:0> ] [ "<STR_LIT>" ] ) != sid : <EOL> user_aid_cnt_rows = next ( user_aid_cnt_iter , ( None , None ) ) [ <NUM_LIT:1> ] <EOL> user_aid_cnt_dict = { } <EOL> for row in user_aid_cnt_rows : <EOL> aid = int ( row [ "<STR_LIT>" ] ) <EOL> user_aid_cnt_dict [ aid ] = row <EOL> ad_infos = [ ] <EOL> for row in rows : <EOL> aid = int ( row [ "<STR_LIT>" ] ) <EOL> row . update ( user_aid_cnt_dict [ aid ] ) <EOL> ad_infos . append ( get_ad_info ( aid ) ) <EOL> uid = int ( sinfo [ "<STR_LIT>" ] ) <EOL> user_info = user_info_map . get ( uid , { "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" } ) <EOL> feature_map [ "<STR_LIT>" ] = [ user_cnt_row ] <EOL> feature_map [ "<STR_LIT>" ] = [ user_info ] <EOL> feature_map [ "<STR_LIT>" ] = ad_infos <EOL> feature_map [ "<STR_LIT>" ] = rows <EOL> feature_map [ "<STR_LIT>" ] = [ sinfo ] <EOL> return feature_map <EOL> def extract_slot_feas ( rows , sinfo ) : <EOL> data = map ( lambda x : ( int ( x [ "<STR_LIT>" ] ) , int ( x [ "<STR_LIT>" ] ) , x ) , rows ) <EOL> data . sort ( ) <EOL> price_data = [ ] <EOL> ot_cnt = defaultdict ( int ) <EOL> all_pos = [ ] <EOL> all_ot = [ ] <EOL> for i in range ( len ( data ) ) : <EOL> all_pos . append ( data [ i ] [ <NUM_LIT:0> ] ) <EOL> all_ot . append ( data [ i ] [ <NUM_LIT:1> ] ) <EOL> aid = int ( data [ i ] [ <NUM_LIT:2> ] [ "<STR_LIT>" ] ) <EOL> price_data . append ( ( get_ad_price ( aid ) , i ) ) <EOL> i_obt = data [ i ] [ <NUM_LIT:1> ] <EOL> ot_cnt [ i_obt ] += <NUM_LIT:1> <EOL> ucnt , lcnt = <NUM_LIT:0> , <NUM_LIT:0> <EOL> for j in range ( len ( data ) ) : <EOL> if i == j : <EOL> continue <EOL> j_obt = data [ j ] [ <NUM_LIT:1> ] <EOL> if j_obt == <NUM_LIT:2> : <EOL> if i < j : <EOL> lcnt += <NUM_LIT:1> <EOL> else : <EOL> ucnt += <NUM_LIT:1> <EOL> data [ i ] [ <NUM_LIT:2> ] [ "<STR_LIT>" ] = lcnt <EOL> data [ i ] [ <NUM_LIT:2> ] [ "<STR_LIT>" ] = ucnt <EOL> for k in range ( <NUM_LIT:1> , <NUM_LIT:4> ) : <EOL> v = ot_cnt [ k ] <EOL> sinfo [ "<STR_LIT>" % k ] = v <EOL> sinfo [ "<STR_LIT>" ] = len ( rows ) <EOL> sinfo [ "<STR_LIT>" ] = hash_val ( <NUM_LIT:0> , tuple ( all_pos ) ) <EOL> sinfo [ "<STR_LIT>" ] = hash_val ( <NUM_LIT:0> , tuple ( all_ot ) ) <EOL> price_data . sort ( ) <EOL> avg_price , avg_cnt = <NUM_LIT:0> , <NUM_LIT:0> <EOL> for p , i in price_data : <EOL> if p != "<STR_LIT>" : <EOL> avg_price += p <EOL> avg_cnt += <NUM_LIT:1> <EOL> data [ i ] [ <NUM_LIT:2> ] [ "<STR_LIT>" ] = i <EOL> else : <EOL> data [ i ] [ <NUM_LIT:2> ] [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> if avg_cnt == <NUM_LIT:0> or avg_price <= <NUM_LIT:0> : <EOL> pass <EOL> else : <EOL> avg_price /= avg_cnt <EOL> for p , i in price_data : <EOL> if not p : <EOL> ratio = - <NUM_LIT:1> <EOL> elif avg_price <= <NUM_LIT:0> : <EOL> ratio = - <NUM_LIT:2> <EOL> else : <EOL> ratio = int ( round ( ( p / avg_price ) * <NUM_LIT:100> ) ) <EOL> data [ i ] [ <NUM_LIT:2> ] [ "<STR_LIT>" ] = ratio <EOL> def stream_info_func ( vs , name = False ) : <EOL> keys = [ "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] <EOL> for v in vs [ <NUM_LIT:0> ] : <EOL> if name : <EOL> yield keys <EOL> else : <EOL> x = { } <EOL> for k in keys : <EOL> val = v [ k ] <EOL> if k == "<STR_LIT>" : <EOL> if val != "<STR_LIT>" : <EOL> val = int ( round ( float ( val ) * <NUM_LIT:1000> ) ) <EOL> elif k in ( "<STR_LIT>" , "<STR_LIT>" ) : <EOL> val = log_trans ( int ( val ) ) <EOL> x [ k ] = val <EOL> x [ "<STR_LIT>" ] = calc_ctr ( int ( x [ "<STR_LIT>" ] ) , int ( x [ "<STR_LIT>" ] ) ) <EOL> yield x <EOL> def sinfo_func ( vs , name = False ) : <EOL> keys = [ <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] <EOL> for v in vs [ <NUM_LIT:0> ] : <EOL> if name : <EOL> yield keys <EOL> else : <EOL> x = { } <EOL> for k in keys : <EOL> val = v [ k ] <EOL> if k == "<STR_LIT>" : <EOL> query = unicode ( v [ "<STR_LIT>" ] , "<STR_LIT:utf-8>" ) <EOL> val = map ( lambda x : hash_val ( <NUM_LIT:0> , x ) , query . split ( ) ) <EOL> if len ( val ) == <NUM_LIT:0> : <EOL> val = [ - <NUM_LIT:1> , ] <EOL> x [ k ] = val <EOL> yield x <EOL> def user_info_func ( vs , name = False ) : <EOL> for v in vs [ <NUM_LIT:0> ] : <EOL> if name : <EOL> yield v . keys ( ) <EOL> else : <EOL> x = { } <EOL> for k in v : <EOL> val = v [ k ] <EOL> x [ k ] = val <EOL> yield x <EOL> def ad_info_func ( vs , name ) : <EOL> keys = [ "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] <EOL> for v in vs [ <NUM_LIT:0> ] : <EOL> if name : <EOL> yield keys <EOL> else : <EOL> x = { } <EOL> for k in keys : <EOL> val = v [ k ] <EOL> if k == "<STR_LIT>" and val != "<STR_LIT>" : <EOL> val = int ( round ( log ( val + <NUM_LIT:1> ) ) ) <EOL> x [ k ] = val <EOL> yield x <EOL> def ngram ( query_word ) : <EOL> sz = len ( query_word ) <EOL> res = [ ] <EOL> for i in range ( sz - <NUM_LIT:1> ) : <EOL> res . append ( u"<STR_LIT>" % ( query_word [ i ] , query_word [ i + <NUM_LIT:1> ] ) ) <EOL> return res <EOL> def calc_sim ( qw , tw ) : <EOL> cnt = <NUM_LIT:0> <EOL> min_pos = <NUM_LIT> <EOL> match_w = [ ] <EOL> for w in qw : <EOL> if w in tw : <EOL> match_w . append ( w ) <EOL> cnt += <NUM_LIT:1> <EOL> min_pos = min ( min_pos , tw . index ( w ) ) <EOL> ratio = int ( round ( ( cnt * <NUM_LIT:1.0> / len ( qw ) ) * <NUM_LIT:100> ) ) <EOL> return { "<STR_LIT>" : cnt , <EOL> "<STR_LIT>" : min_pos , <EOL> "<STR_LIT>" : ratio , <EOL> "<STR_LIT>" : match_w } <EOL> def query_feas ( query_word , ad_info , name = False ) : <EOL> if name : <EOL> return [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> x = { } <EOL> title = ad_info [ "<STR_LIT>" ] . split ( ) <EOL> x [ "<STR_LIT>" ] = len ( title ) <EOL> if "<STR_LIT:U+0020>" . join ( query_word ) in ad_info [ "<STR_LIT>" ] : <EOL> x [ "<STR_LIT>" ] = <NUM_LIT:1> <EOL> else : <EOL> x [ "<STR_LIT>" ] = <NUM_LIT:0> <EOL> title = ad_info [ "<STR_LIT>" ] <EOL> title_val = map ( lambda x : hash_val ( <NUM_LIT:0> , x ) , title . split ( ) ) <EOL> x [ "<STR_LIT:title>" ] = title_val if title_val else [ - <NUM_LIT:1> , ] <EOL> if len ( query_word ) == <NUM_LIT:0> : <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> else : <EOL> sim = calc_sim ( query_word , title ) <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> qw_ngram = ngram ( query_word ) <EOL> if len ( qw_ngram ) == <NUM_LIT:0> : <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> else : <EOL> title_ngram = ngram ( title ) <EOL> sim = calc_sim ( qw_ngram , title_ngram ) <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> x [ "<STR_LIT>" ] = sim [ "<STR_LIT>" ] <EOL> return x <EOL> unmatch_set = set ( ) <EOL> def param_feas ( se_params , ad_info , name = False ) : <EOL> if name : <EOL> return [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> if len ( se_params ) == <NUM_LIT:0> : <EOL> return [ - <NUM_LIT:1> , - <NUM_LIT:1> , - <NUM_LIT:1> ] <EOL> ad_params = ad_info [ "<STR_LIT>" ] <EOL> x = { } <EOL> par_match = <NUM_LIT:0> <EOL> par_miss = <NUM_LIT:0> <EOL> par_nmatch = <NUM_LIT:0> <EOL> for par_key , par_v in se_params . items ( ) : <EOL> ad_v = ad_params . get ( par_key ) <EOL> if par_v == ad_v : <EOL> par_match += <NUM_LIT:1> <EOL> elif ad_v is not None : <EOL> par_nmatch += <NUM_LIT:1> <EOL> key = ( type ( par_v ) , type ( ad_v ) ) <EOL> if key not in unmatch_set : <EOL> unmatch_set . add ( key ) <EOL> print key <EOL> print par_v . encode ( "<STR_LIT:utf-8>" ) , u"<STR_LIT>" , ad_v . encode ( "<STR_LIT:utf-8>" ) <EOL> else : <EOL> par_miss += <NUM_LIT:1> <EOL> x [ "<STR_LIT>" ] = par_match <EOL> x [ "<STR_LIT>" ] = par_nmatch <EOL> x [ "<STR_LIT>" ] = par_miss <EOL> return x <EOL> def match_info_func ( vs , name ) : <EOL> ad_infos = vs [ <NUM_LIT:0> ] <EOL> sinfo = vs [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] <EOL> s_ca_id = int ( sinfo . get ( "<STR_LIT>" ) ) <EOL> s_ca_pid = category_map [ s_ca_id ] <EOL> query = unicode ( sinfo [ "<STR_LIT>" ] , "<STR_LIT:utf-8>" ) <EOL> query = query . split ( ) if query else [ ] <EOL> keys = [ <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" <EOL> ] + query_feas ( None , None , True ) <EOL> for ad_info in ad_infos : <EOL> if name : <EOL> yield keys <EOL> else : <EOL> x = { } <EOL> """<STR_LIT>""" <EOL> ca_id = int ( ad_info . get ( "<STR_LIT>" , - <NUM_LIT:1> ) ) <EOL> if ca_id == s_ca_id : <EOL> x [ "<STR_LIT>" ] = ca_id <EOL> else : <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> ca_pid = category_map . get ( ca_id , - <NUM_LIT:1> ) <EOL> if ca_pid == s_ca_pid : <EOL> x [ "<STR_LIT>" ] = ca_pid <EOL> else : <EOL> x [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> x . update ( query_feas ( query , ad_info ) ) <EOL> yield x <EOL> log_cnt_keys = set ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> def user_cnt_func ( vs , name ) : <EOL> keys = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] <EOL> for v in vs [ <NUM_LIT:0> ] : <EOL> if name : <EOL> yield keys <EOL> else : <EOL> x = { } <EOL> for k in keys : <EOL> if k not in v : <EOL> continue <EOL> val = v [ k ] <EOL> if k in log_cnt_keys : <EOL> val = log_trans ( int ( val ) ) <EOL> x [ k ] = val <EOL> x [ "<STR_LIT>" ] = calc_ctr ( int ( v [ "<STR_LIT>" ] ) , int ( v [ "<STR_LIT>" ] ) ) <EOL> yield x <EOL> extract_func = [ <EOL> ( stream_info_func , [ "<STR_LIT>" ] ) , <EOL> ( sinfo_func , [ "<STR_LIT>" ] ) , <EOL> ( user_info_func , [ "<STR_LIT>" ] ) , <EOL> ( ad_info_func , [ "<STR_LIT>" ] ) , <EOL> ( user_cnt_func , [ "<STR_LIT>" ] ) , <EOL> ( match_info_func , [ "<STR_LIT>" , "<STR_LIT>" ] ) , <EOL> ] <EOL> def extract ( feature_map , name = False ) : <EOL> ins_size = <NUM_LIT:0> <EOL> for k , v in feature_map . items ( ) : <EOL> ins_size = max ( ins_size , len ( v ) ) <EOL> instances = [ { } for _ in xrange ( ins_size ) ] <EOL> for func , in_keys in extract_func : <EOL> vls = map ( lambda k : feature_map [ k ] , in_keys ) <EOL> msize = reduce ( lambda x , y : max ( x , len ( y ) ) , vls , <NUM_LIT:0> ) <EOL> if msize == <NUM_LIT:1> : <EOL> for x in func ( vls , name ) : <EOL> for ins in instances : <EOL> ins . update ( x ) <EOL> break <EOL> else : <EOL> for t , x in enumerate ( func ( vls , name ) ) : <EOL> instances [ t ] . update ( x ) <EOL> return instances <EOL> one_hot_encoder = defaultdict ( int ) <EOL> def hash_val ( t , v , dtype = None , D = <NUM_LIT> ) : <EOL> if dtype == "<STR_LIT>" : <EOL> return u"<STR_LIT>" % ( t , v ) <EOL> elif dtype == "<STR_LIT>" : <EOL> v = ( t << D ) | ( hash ( unicode ( v ) ) & ( ( <NUM_LIT:1> << D ) - <NUM_LIT:1> ) ) <EOL> if v in one_hot_encoder : <EOL> return u"<STR_LIT>" % ( one_hot_encoder [ v ] + <NUM_LIT> , <NUM_LIT:1> ) <EOL> else : <EOL> one_hot_encoder [ v ] = len ( one_hot_encoder ) <EOL> return u"<STR_LIT>" % ( one_hot_encoder [ v ] + <NUM_LIT> , <NUM_LIT:1> ) <EOL> else : <EOL> return ( t << D ) | ( hash ( unicode ( v ) ) & ( ( <NUM_LIT:1> << D ) - <NUM_LIT:1> ) ) <EOL> def main ( ) : <EOL> random . seed ( args . seed ) <EOL> xgb_set = set ( [ <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> ] ) <EOL> xgb_sparse_set = set ( [ <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" <EOL> ] ) <EOL> if args . test : <EOL> fh_list = [ open ( "<STR_LIT>" % ( args . test , args . type ) , "<STR_LIT:w>" ) , <EOL> open ( "<STR_LIT>" % ( args . test , args . type ) , "<STR_LIT:w>" ) , <EOL> open ( "<STR_LIT>" % ( args . test , args . type ) , "<STR_LIT:w>" ) ] <EOL> else : <EOL> fh_list = [ open ( "<STR_LIT>" % ( args . type ) , "<STR_LIT:w>" ) , <EOL> open ( "<STR_LIT>" % ( args . type ) , "<STR_LIT:w>" ) , <EOL> open ( "<STR_LIT>" % ( args . type ) , "<STR_LIT:w>" ) ] <EOL> data_iter = data ( args . test , maxlines = args . maxl ) <EOL> print "<STR_LIT>" % args . sr <EOL> avg_ctr = defaultdict ( lambda : [ <NUM_LIT:0> , <NUM_LIT:0> ] ) <EOL> for line_cnt , ( data_type , rows , sinfo ) in enumerate ( data_iter ) : <EOL> sinfo [ "<STR_LIT>" ] = int ( sinfo [ "<STR_LIT>" ] ) <EOL> sinfo [ "<STR_LIT>" ] = int ( sinfo [ "<STR_LIT>" ] ) <EOL> extract_slot_feas ( rows , sinfo ) <EOL> rows = filter ( lambda x : filter_row ( x , data_type , sr = args . sr ) , rows ) <EOL> if not rows : <EOL> continue <EOL> feature_map = get_features ( sinfo , rows , data_type > <NUM_LIT:0> ) <EOL> instances = extract ( feature_map ) <EOL> if line_cnt == <NUM_LIT:0> : <EOL> for k , feas in feature_map . items ( ) : <EOL> print "<STR_LIT:->" * <NUM_LIT> <EOL> print k <EOL> print feas [ <NUM_LIT:0> ] . keys ( ) <EOL> feas_name = sorted ( instances [ <NUM_LIT:0> ] . keys ( ) ) <EOL> print len ( feas_name ) , feas_name <EOL> if args . sz is not None : <EOL> write_dump ( "<STR_LIT>" , feas_name ) <EOL> elif args . test : <EOL> write_dump ( "<STR_LIT>" % args . test , feas_name ) <EOL> else : <EOL> write_dump ( "<STR_LIT>" , feas_name ) <EOL> fh = fh_list [ data_type ] <EOL> for ins_map , row in zip ( instances , rows ) : <EOL> y = int ( row . get ( "<STR_LIT>" , <NUM_LIT:0> ) ) <EOL> avg_ctr [ data_type ] [ <NUM_LIT:0> ] += y <EOL> avg_ctr [ data_type ] [ <NUM_LIT:1> ] += <NUM_LIT:1> <EOL> ins = [ ] <EOL> for kt , k in enumerate ( feas_name ) : <EOL> if "<STR_LIT>" in args . type : <EOL> if k in xgb_set : <EOL> hash_type = "<STR_LIT>" <EOL> elif k in xgb_sparse_set : <EOL> hash_type = "<STR_LIT>" <EOL> else : <EOL> if line_cnt == <NUM_LIT:0> : <EOL> print "<STR_LIT>" % k <EOL> continue <EOL> else : <EOL> hash_type = "<STR_LIT>" <EOL> feas = ins_map [ k ] <EOL> if line_cnt == <NUM_LIT:0> : <EOL> print kt , k , type ( feas ) , feas <EOL> if isinstance ( feas , list ) or isinstance ( feas , tuple ) : <EOL> for f in feas : <EOL> ins . append ( hash_val ( kt + <NUM_LIT:1> , f , hash_type ) ) <EOL> else : <EOL> ins . append ( hash_val ( kt + <NUM_LIT:1> , feas , hash_type ) ) <EOL> fh . write ( unicode ( y ) + "<STR_LIT:U+0020>" + "<STR_LIT:U+0020>" . join ( map ( unicode , ins ) ) + "<STR_LIT:\n>" ) <EOL> for key , value in avg_ctr . items ( ) : <EOL> print "<STR_LIT>" % ( key , value [ <NUM_LIT:0> ] * <NUM_LIT:1.> / value [ <NUM_LIT:1> ] ) <EOL> for fh in fh_list : <EOL> fh . close ( ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , type = int , default = <NUM_LIT:0> ) <EOL> parser . add_argument ( '<STR_LIT>' , type = int , default = <NUM_LIT:0> ) <EOL> parser . add_argument ( '<STR_LIT>' , type = int , default = None ) <EOL> parser . add_argument ( '<STR_LIT>' , type = int , default = <NUM_LIT> ) <EOL> parser . add_argument ( '<STR_LIT>' , type = str , default = "<STR_LIT>" ) <EOL> parser . add_argument ( '<STR_LIT>' , type = float , default = <NUM_LIT:0.1> ) <EOL> parser . add_argument ( '<STR_LIT>' , type = int , default = <NUM_LIT:9> ) <EOL> parser . add_argument ( '<STR_LIT>' , type = int , default = <NUM_LIT:0> ) <EOL> parser . add_argument ( '<STR_LIT>' , type = int , default = <NUM_LIT:1> ) <EOL> args = parser . parse_args ( ) <EOL> if args . mongo : <EOL> from pymongo import MongoClient <EOL> import functools32 as functools <EOL> client = MongoClient ( '<STR_LIT:localhost>' , <NUM_LIT> ) <EOL> db = client . test <EOL> @ functools . lru_cache ( maxsize = <NUM_LIT> ) <EOL> def get_ad_info ( aid ) : <EOL> ad_info = db . ad_info . find_one ( { "<STR_LIT>" : aid } ) <EOL> return trans_ad_info ( ad_info ) <EOL> user_info_map = get_user_info ( ) <EOL> category_map = get_category ( ) <EOL> user_cnt_iter = read_tsv ( "<STR_LIT>" , delimiter = "<STR_LIT:U+002C>" ) <EOL> user_aid_cnt_iter = next_row ( read_tsv ( "<STR_LIT>" , delimiter = "<STR_LIT:U+002C>" ) ) <EOL> main ( ) </s>
