<s> import mysql <EOL> import sys <EOL> __author__ = '<STR_LIT>' <EOL> reload ( sys ) <EOL> sys . setdefaultencoding ( "<STR_LIT:utf-8>" ) <EOL> import datetime <EOL> import re <EOL> db_goods_unique_key = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> class CrawlDao : <EOL> html_tag = re . compile ( r'<STR_LIT>' , re . S ) <EOL> def __init__ ( self ) : <EOL> conf = mysql . get_local_conf ( ) <EOL> self . db = mysql . PDBC ( conf ) <EOL> def get_attr_key_id ( self , source_id , key , cate_id ) : <EOL> sql = '<STR_LIT>' + str ( <EOL> source_id ) + '<STR_LIT>' + key + '<STR_LIT>' + str ( cate_id ) <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = source_id <EOL> db_dict [ '<STR_LIT:name>' ] = key <EOL> db_dict [ '<STR_LIT>' ] = cate_id <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def get_attr_value_id ( self , source_id , key_id , value ) : <EOL> sql = '<STR_LIT>' + str ( source_id ) + '<STR_LIT>' + str ( <EOL> key_id ) + '<STR_LIT>' + value + '<STR_LIT:">' <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = source_id <EOL> db_dict [ '<STR_LIT:name>' ] = value <EOL> db_dict [ '<STR_LIT>' ] = key_id <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def get_goods_id ( self , source_id , goods_id ) : <EOL> sql = '<STR_LIT>' + str ( source_id ) + '<STR_LIT>' + str ( <EOL> goods_id ) + '<STR_LIT:">' <EOL> data = self . db . get_data ( sql ) <EOL> if len ( data ) == <NUM_LIT:0> : <EOL> return False <EOL> else : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> def get_brand_id ( self , source_id , name ) : <EOL> sql = '<STR_LIT>' + str ( source_id ) + '<STR_LIT>' + name + '<STR_LIT:">' <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = source_id <EOL> db_dict [ '<STR_LIT:name>' ] = name <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def insert_temple ( self , table , dic , replace = False ) : <EOL> gmt = datetime . datetime . now ( ) . strftime ( "<STR_LIT>" ) <EOL> dic [ '<STR_LIT>' ] = gmt <EOL> dic [ '<STR_LIT>' ] = gmt <EOL> if replace : <EOL> action = '<STR_LIT:replace>' <EOL> else : <EOL> action = '<STR_LIT>' <EOL> sql = action + '<STR_LIT>' + table + '<STR_LIT:(>' + '<STR_LIT:U+002C>' . join ( dic . keys ( ) ) + '<STR_LIT>' <EOL> value_list = list ( ) <EOL> for key , value in dic . items ( ) : <EOL> value = str ( value ) . replace ( '<STR_LIT:">' , '<STR_LIT>' ) <EOL> value_list . append ( '<STR_LIT:">' + self . html_tag . sub ( '<STR_LIT>' , str ( value ) ) + '<STR_LIT:">' ) <EOL> sql += '<STR_LIT:(>' + '<STR_LIT:U+002C>' . join ( value_list ) + '<STR_LIT:)>' <EOL> print sql <EOL> return self . db . insert_data ( sql ) <EOL> def insert_or_update_goods ( self , dic ) : <EOL> select_sql = '<STR_LIT>' + str ( <EOL> dic [ '<STR_LIT>' ] ) + '<STR_LIT>' + dic [ '<STR_LIT>' ] + '<STR_LIT:">' <EOL> data = self . db . get_data ( select_sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> return self . insert_temple ( "<STR_LIT>" , dic ) <EOL> def insert_or_update_goods_attr ( self , dic ) : <EOL> condition_list = list ( ) <EOL> for key , value in dic . items ( ) : <EOL> condition_list . append ( key + "<STR_LIT>" + str ( value ) ) <EOL> sql = '<STR_LIT>' + '<STR_LIT>' . join ( condition_list ) <EOL> data = self . db . get_data ( sql ) <EOL> if len ( data ) > <NUM_LIT:0> : <EOL> return <EOL> self . insert_temple ( "<STR_LIT>" , dic ) <EOL> def insert_batch_temple ( self , table , batch_list , is_replace = False ) : <EOL> gmt = datetime . datetime . now ( ) . strftime ( "<STR_LIT>" ) <EOL> for i in batch_list : <EOL> i [ '<STR_LIT>' ] = gmt <EOL> i [ '<STR_LIT>' ] = gmt <EOL> action = '<STR_LIT>' <EOL> if is_replace : <EOL> action = '<STR_LIT:replace>' <EOL> sql = action + '<STR_LIT>' + table + '<STR_LIT:(>' + '<STR_LIT:U+002C>' . join ( batch_list [ <NUM_LIT:0> ] . keys ( ) ) + '<STR_LIT>' <EOL> value_list = list ( ) <EOL> for i in batch_list : <EOL> inner_value_list = list ( ) <EOL> for key , value in i . items ( ) : <EOL> if isinstance ( value , str ) or isinstance ( value , unicode ) : <EOL> inner_value_list . append ( '<STR_LIT:">' + value + '<STR_LIT:">' ) <EOL> else : <EOL> inner_value_list . append ( str ( value ) ) <EOL> value_list . append ( '<STR_LIT:(>' + '<STR_LIT:U+002C>' . join ( inner_value_list ) + '<STR_LIT:)>' ) <EOL> sql += '<STR_LIT:U+002C>' . join ( value_list ) <EOL> self . db . exec_data ( sql ) <EOL> def get_car_id ( self , source_id , level , parent_id , name , remarks = None ) : <EOL> sql = '<STR_LIT>' + str ( source_id ) + '<STR_LIT>' + str ( <EOL> level ) + '<STR_LIT>' + str ( parent_id ) + '<STR_LIT>' + name + '<STR_LIT:">' <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> insert_dic = dict ( ) <EOL> insert_dic [ '<STR_LIT>' ] = str ( level ) <EOL> insert_dic [ '<STR_LIT>' ] = str ( parent_id ) <EOL> insert_dic [ '<STR_LIT:name>' ] = name <EOL> insert_dic [ '<STR_LIT>' ] = str ( source_id ) <EOL> if remarks : <EOL> insert_dic [ '<STR_LIT>' ] = remarks <EOL> return self . insert_temple ( "<STR_LIT>" , insert_dic ) <EOL> def get_car_type ( self , source_id , car_dic ) : <EOL> car_id_list = list ( ) <EOL> brand_id = self . get_car_id ( source_id , <NUM_LIT:1> , <NUM_LIT:0> , car_dic [ '<STR_LIT>' ] ) <EOL> car_id_list . append ( brand_id ) <EOL> series_id = self . get_car_id ( source_id , <NUM_LIT:2> , brand_id , car_dic [ '<STR_LIT>' ] ) <EOL> car_id_list . append ( series_id ) <EOL> level = <NUM_LIT:3> <EOL> parent_id = series_id <EOL> key_list = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> for key in key_list : <EOL> if not key in car_dic . keys ( ) : <EOL> break <EOL> car_id = self . get_car_id ( source_id , level , parent_id , car_dic [ key ] ) <EOL> car_id_list . append ( car_id ) <EOL> parent_id = car_id <EOL> level += <NUM_LIT:1> <EOL> return car_id_list <EOL> def save_goods_price ( self , source_id , goods_id , price ) : <EOL> goods_price_dic = dict ( ) <EOL> goods_price_dic [ '<STR_LIT>' ] = source_id <EOL> goods_price_dic [ '<STR_LIT>' ] = goods_id <EOL> goods_price_dic [ '<STR_LIT>' ] = price <EOL> goods_price_dic [ '<STR_LIT>' ] = datetime . datetime . now ( ) . strftime ( "<STR_LIT>" ) <EOL> self . insert_temple ( "<STR_LIT>" , goods_price_dic , True ) <EOL> def get_all_special_sign ( self ) : <EOL> sql = '<STR_LIT>' <EOL> data = self . db . get_data ( sql ) <EOL> special_list = list ( ) <EOL> for i in data : <EOL> special_list . append ( i [ '<STR_LIT>' ] ) <EOL> return special_list <EOL> def get_cate_by_source_pid ( self , source_id , pid_list ) : <EOL> if not pid_list : <EOL> return None <EOL> tmp_set = set ( ) <EOL> for e in pid_list : <EOL> tmp_set . add ( str ( e ) ) <EOL> sql = '<STR_LIT>' + str ( <EOL> source_id ) + '<STR_LIT>' + '<STR_LIT:U+002C>' . join ( tmp_set ) + '<STR_LIT:)>' <EOL> data = self . db . get_data ( sql ) <EOL> return data <EOL> def get_cate_id_by_name ( self , source_cate_id , source_id , name ) : <EOL> sql = '<STR_LIT>' + str ( source_id ) + '<STR_LIT>' + str ( <EOL> source_cate_id ) + '<STR_LIT>' + name + '<STR_LIT:">' <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> insert_dict = dict ( ) <EOL> insert_dict [ '<STR_LIT>' ] = source_cate_id <EOL> insert_dict [ '<STR_LIT>' ] = str ( source_id ) <EOL> insert_dict [ '<STR_LIT:name>' ] = name <EOL> return self . insert_temple ( "<STR_LIT>" , insert_dict ) <EOL> def get_cate_by_source ( self , source_id ) : <EOL> sql = '<STR_LIT>' '<STR_LIT>' % source_id <EOL> data = self . db . get_data ( sql ) <EOL> return data <EOL> def get_cate_by_cateids ( self , source_id , cate_ids ) : <EOL> ids = [ ] <EOL> for id in cate_ids : <EOL> if isinstance ( id , str ) or isinstance ( id , unicode ) : <EOL> ids . append ( id ) <EOL> else : <EOL> ids . append ( str ( id ) ) <EOL> sql_select = '<STR_LIT>' + str ( <EOL> source_id ) + '<STR_LIT>' + '<STR_LIT:U+002C>' . join ( ids ) + '<STR_LIT:)>' <EOL> return self . db . get_data ( sql_select ) <EOL> def get_source_id ( self , name ) : <EOL> select_sql = '<STR_LIT>' + name + '<STR_LIT:">' <EOL> data = self . db . get_data ( select_sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT:name>' ] = name <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def get_store_id ( self , db_dict ) : <EOL> select_sql = '<STR_LIT>' + str ( db_dict [ '<STR_LIT>' ] ) + '<STR_LIT>' + db_dict [ '<STR_LIT:name>' ] + '<STR_LIT:">' <EOL> data = self . db . get_data ( select_sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def insert_goods_store ( self , source_id , goods_id , store_id ) : <EOL> select_sql = '<STR_LIT>' + str ( goods_id ) <EOL> data = self . db . get_data ( select_sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = source_id <EOL> db_dict [ '<STR_LIT>' ] = goods_id <EOL> db_dict [ '<STR_LIT>' ] = store_id <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def insert_or_update_template ( self , table , db_dict , unique_key , primary_name = '<STR_LIT:id>' ) : <EOL> select_sql = str ( ) <EOL> for key in unique_key : <EOL> select_sql += key + '<STR_LIT>' + str ( db_dict [ key ] ) + '<STR_LIT>' <EOL> primary_key_id = <NUM_LIT:0> <EOL> if select_sql : <EOL> select_sql = '<STR_LIT>' % ( table , select_sql [ : - <NUM_LIT:5> ] ) <EOL> data = self . db . get_data ( select_sql ) <EOL> if data : <EOL> primary_key_id = data [ <NUM_LIT:0> ] [ primary_name ] <EOL> if primary_key_id > <NUM_LIT:0> : <EOL> gmt = datetime . datetime . now ( ) . strftime ( "<STR_LIT>" ) <EOL> db_dict [ '<STR_LIT>' ] = gmt <EOL> value_list = list ( ) <EOL> for key , value in db_dict . items ( ) : <EOL> value_list . append ( '<STR_LIT>' % ( key , str ( value ) ) ) <EOL> self . db . exec_data ( '<STR_LIT>' % ( <EOL> table , '<STR_LIT:U+002C>' . join ( value_list ) , '<STR_LIT>' % ( primary_name , primary_key_id ) ) ) <EOL> else : <EOL> primary_key_id = self . insert_temple ( table , db_dict ) <EOL> return primary_key_id <EOL> def insert_car_goods ( self , source_id , car_id , goods_id ) : <EOL> sql = '<STR_LIT>' + str ( source_id ) + '<STR_LIT>' + str ( car_id ) + '<STR_LIT>' + str ( goods_id ) <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> else : <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = source_id <EOL> db_dict [ '<STR_LIT>' ] = goods_id <EOL> db_dict [ '<STR_LIT>' ] = car_id <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def get_car_service_id ( self , source_id , source_service_id , cate , name , suggest ) : <EOL> sql = '<STR_LIT>' + str ( <EOL> source_id ) + '<STR_LIT>' + cate + '<STR_LIT>' + name + '<STR_LIT:">' <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = str ( source_id ) <EOL> db_dict [ '<STR_LIT>' ] = cate <EOL> db_dict [ '<STR_LIT:name>' ] = name <EOL> db_dict [ '<STR_LIT>' ] = suggest <EOL> if source_service_id : <EOL> db_dict [ '<STR_LIT>' ] = source_service_id <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def get_car_run_service_id ( self , source_id , car_id , firsttime , mile , service_id , service_type , <EOL> service_suggest = None ) : <EOL> sql = '<STR_LIT>' + str ( source_id ) + '<STR_LIT>' + str ( <EOL> car_id ) + '<STR_LIT>' + firsttime + '<STR_LIT>' + str ( mile ) + "<STR_LIT>" + str ( service_id ) <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = str ( source_id ) <EOL> db_dict [ '<STR_LIT>' ] = str ( car_id ) <EOL> db_dict [ '<STR_LIT>' ] = firsttime <EOL> db_dict [ '<STR_LIT>' ] = str ( mile ) <EOL> db_dict [ '<STR_LIT>' ] = str ( service_id ) <EOL> db_dict [ '<STR_LIT>' ] = str ( service_type ) <EOL> if service_suggest : <EOL> db_dict [ '<STR_LIT>' ] = service_suggest <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) <EOL> def add_car_run_service_goods ( self , source_id , car_run_service_id , source_goods_id , goods_id = None ) : <EOL> sql = '<STR_LIT>' + str ( <EOL> source_id ) + '<STR_LIT>' + str ( <EOL> car_run_service_id ) + '<STR_LIT>' + source_goods_id + '<STR_LIT:">' <EOL> data = self . db . get_data ( sql ) <EOL> if data : <EOL> return data [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] <EOL> db_dict = dict ( ) <EOL> db_dict [ '<STR_LIT>' ] = str ( source_id ) <EOL> db_dict [ '<STR_LIT>' ] = str ( car_run_service_id ) <EOL> db_dict [ '<STR_LIT>' ] = source_goods_id <EOL> if goods_id : <EOL> db_dict [ '<STR_LIT>' ] = str ( goods_id ) <EOL> return self . insert_temple ( '<STR_LIT>' , db_dict ) </s>
