BOT_NAME = 'fooltrader'
SPIDER_MODULES = ['fooltrader.spiders']
NEWSPIDER_MODULE = 'fooltrader.spiders'
COMPRESSION_ENABLED = True
DOWNLOADER_MIDDLEWARES = {
    'fooltrader.middlewares.ForbiddenHandleMiddleware': 1000,
}
AUTOTHROTTLE_DEBUG = False
FOOLTRADER_STORE_PATH = os.environ.get('FOOLTRADER_STORE_PATH')
    FOOLTRADER_STORE_PATH = '/home/xuanqi/workspace/github/fooltrader/data'
STOCK_START_CODE = '000001'
STOCK_END_CODE = '666666'
US_STOCK_CODES = ['MSFT', 'AMZN', 'NVDA', 'SINA', 'AAPL']
AVAILABLE_TICK_DATE = '2004-10-8'
START_TICK_DATE = '2013-1-1'
FORCE_DOWNLOAD_SECURITY_LIST = False
DOWNLOAD_TXT_ENCODING = 'GB2312'
AUTO_KAFKA = False
KAFKA_HOST = 'localhost:9092'
KAFKA_PATH = '/home/xuanqi/software/kafka_2.11-0.11.0.1'
ZK_KAFKA_HOST = 'localhost:2181'
DG_PATH = '/Users/xuanqi/software/delegate/DGROOT/bin/delegate'
SUPPORT_SOCKS2HTTP = False
TIME_FORMAT_MICRO = "%Y%m%d %H%M%S.%f"
TIME_FORMAT_SEC = '%Y-%m-%d %H:%M:%S'
TIME_FORMAT_DAY = '%Y-%m-%d'
ES_HOSTS = ['localhost:9200']
SMTP_HOST = 'smtpdm.aliyun.com'
SMTP_PORT = '80'
EMAIL_USER_NAME = ''
    EMAIL_USER_NAME = os.environ.get('EMAIL_USER_NAME')
EMAIL_PASSWORD = ''
    EMAIL_PASSWORD = os.environ.get('EMAIL_PASSWORD')
WEIXIN_APP_ID = ""
    WEIXIN_APP_ID = os.environ.get("WEIXIN_APP_ID")
WEIXIN_APP_SECRECT = ""
    WEIXIN_APP_SECRECT = os.environ.get("WEIXIN_APP_SECRECT")
EOS_MONGODB_URL = ""
    EOS_MONGODB_URL = os.environ.get("EOS_MONGODB_URL") def get_security_list(security_type='stock', exchanges=None, start_code=None, end_code=None,
                      mode='simple', start_list_date=None, codes=None):
    """
    get security list.

    Parameters
    ----------
    security_type : str
        {‘stock’, 'future'},default: stock
    exchanges : str or list
        ['sh', 'sz','nasdaq','nyse','amex','shfe','dce','zce'],default: ['sh','sz']
    start_code : str
        the start code,work with end,default:None
        if using codes,it would be ignored
    end_code : str
        the end code,works with start,default:None
        if using codes,it would be ignored
    mode : str
        whether parse more security info,{'simple','es'},default:'simple'
    start_list_date : Timestamp str or Timestamp
        the filter for start list date,default:None
    codes : list
        the exact codes to query,default:None

    Returns
    -------
    DataFrame
        the security list

    """
    df = pd.DataFrame()
    if type(exchanges) == str:
        exchanges = [exchanges]

    if not exchanges:
        exchanges = SECURITY_TYPE_MAP_EXCHANGES[security_type]

    if security_type == 'index':
        df = df.append(pd.DataFrame(CHINA_STOCK_SH_INDEX), ignore_index=True)
        df = df.append(pd.DataFrame(CHINA_STOCK_SZ_INDEX), ignore_index=True)
        df = df.append(pd.DataFrame(USA_STOCK_NASDAQ_INDEX), ignore_index=True)
    else:
        for exchange in exchanges:
            the_path = get_security_list_path(security_type, exchange)
            if os.path.exists(the_path):
                converters = None

                if mode == 'es' and security_type == 'stock':
                    converters = {'code': str,
                                  'sinaIndustry': convert_to_list_if_need,
                                  'sinaConcept': convert_to_list_if_need,
                                  'sinaArea': convert_to_list_if_need}
                if converters:
                    df = df.append(pd.read_csv(the_path, converters=converters), ignore_index=True)
                else:
                    df = df.append(pd.read_csv(the_path, dtype=str), ignore_index=True)

    if not df.empty > 0:
        df = df_for_date_range(df, start_date=start_list_date)

        df = df.set_index(df['code'], drop=False)
        df = df.sort_index()

        if codes:
            df = df[df["code"].isin(codes)]
        elif start_code and end_code:
            df = df[(df["code"] >= start_code) & (df["code"] <= end_code)]

        if security_type != 'cryptocurrency':
            df = df.drop_duplicates(subset='code', keep='last')

    return df def get_trading_dates(security_item, dtype='list', ignore_today=False, source='163', fuquan='bfq'):
    df = get_kdata(security_item, source=source, fuquan=fuquan)
    if dtype is 'list' and len(df.index) > 0:
        dates = df.index.strftime('%Y-%m-%d').tolist()
        if ignore_today:
            dates = [the_date for the_date in dates if the_date != datetime.datetime.today().strftime('%Y-%m-%d')]
            return dates
        return dates
    return df.index def get_kdata(security_item, exchange=None, the_date=None, start_date=None, end_date=None, fuquan='bfq', source=None,
              level='day', generate_id=False):
    """
    get kdata.

    Parameters
    ----------
    security_item : SecurityItem or str
        the security item,id or code

    exchange : str
        the exchange,set this for cryptocurrency

    the_date : TimeStamp str or TimeStamp
        get the kdata for the exact date
    start_date : TimeStamp str or TimeStamp
        start date
    end_date : TimeStamp str or TimeStamp
        end date
    fuquan : str
        {"qfq","hfq","bfq"},default:"bfq"
    source : str
        the data source,{'163','sina','exchange'},just used for internal merge
    level : str or int
        the kdata level,{1,5,15,30,60,'day','week','month'},default : 'day'

    Returns
    -------
    DataFrame

    """

    # 由于数字货币的交易所太多，必须指定exchange
    security_item = to_security_item(security_item, exchange)

    source = adjust_source(security_item, source)

    # 163的数据是合并过的,有复权因子,都存在'bfq'目录下,只需从一个地方取数据,并做相应转换
    if source == '163':
        the_path = files_contract.get_kdata_path(security_item, source=source, fuquan='bfq')
    else:
        the_path = files_contract.get_kdata_path(security_item, source=source, fuquan=fuquan)

    if os.path.isfile(the_path):
        df = pd_utils.pd_read_csv(the_path, generate_id=generate_id)

        if 'factor' in df.columns and source == '163' and security_item['type'] == 'stock':
            df_kdata_has_factor = df[df['factor'].notna()]
            if df_kdata_has_factor.shape[0] > 0:
                latest_factor = df_kdata_has_factor.tail(1).factor.iat[0]
            else:
                latest_factor = None

        if the_date:
            if the_date in df.index:
                df = df.loc[the_date:the_date, :]
            else:
                return None
        else:
            if start_date or end_date:
                df = df_for_date_range(df, start_date=start_date, end_date=end_date)

        # 复权处理
        if source == '163' and security_item['type'] == 'stock':
            if 'factor' in df.columns:
                # 后复权是不变的
                df['hfqClose'] = df.close * df.factor
                df['hfqOpen'] = df.open * df.factor
                df['hfqHigh'] = df.high * df.factor
                df['hfqLow'] = df.low * df.factor

                # 前复权需要根据最新的factor往回算,当前价格不变
                if latest_factor:
                    df['qfqClose'] = df.hfqClose / latest_factor
                    df['qfqOpen'] = df.hfqOpen / latest_factor
                    df['qfqHigh'] = df.hfqHigh / latest_factor
                    df['qfqLow'] = df.hfqLow / latest_factor
                else:
                    logger.exception("missing latest factor for {}".format(security_item['id']))

        return df
    return pd.DataFrame() def get_tick_path(item, date):
    return os.path.join(get_tick_dir(item), date + ".csv") STOCK_START_CODE = '000001' STOCK_END_CODE = '666666' def get_datetime(str):
    return datetime.datetime.strptime(str, TIME_FORMAT_DAY) def kdata_to_tick(kdata_json):
    str = '''成交时间	成交价	价格变动	成交量(手)	成交额(元)	性质
{}	{}	--	{}	{}	{}'''.format('09:25:00', kdata_json['high'], int(kdata_json['volume']) / 100,
                                           kdata_json['turnover'], '买盘')
    return str def sina_tick_to_csv(security_item, the_content, the_date):
    csv_path = get_tick_path(security_item, the_date)
    df = read_csv(the_content, "GB2312", sep='\s+')
    df = df.loc[:, ['成交时间', '成交价', '成交量(手)', '成交额(元)', '性质']]
    df.columns = TICK_COL
    df['direction'] = df['direction'].apply(lambda x: direction_to_int(x))
    df.to_csv(csv_path, index=False)
import io
import os
import scrapy
from scrapy import Request
from scrapy import signals
from fooltrader import settings
from fooltrader.api.technical import get_security_list, get_trading_dates, get_kdata
from fooltrader.consts import DEFAULT_TICK_HEADER
from fooltrader.contract.files_contract import get_tick_path
from fooltrader.settings import STOCK_START_CODE, STOCK_END_CODE
from fooltrader.utils.utils import get_datetime, kdata_to_tick, sina_tick_to_csv
# -*- coding: utf-8 -*-





class StockTickSpider(scrapy.Spider):
    name = "stock_tick"

    custom_settings = {
        'DOWNLOAD_DELAY': 2,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 6,

        'SPIDER_MIDDLEWARES': {
            'fooltrader.middlewares.FoolErrorMiddleware': 1000,
        }
    }

    def yield_request(self, item, trading_dates=None):
        if not trading_dates:
            trading_dates = get_trading_dates(item)

        for trading_date in trading_dates:
            if get_datetime(trading_date) < get_datetime(settings.START_TICK_DATE) or get_datetime(
                    trading_date) < get_datetime(settings.AVAILABLE_TICK_DATE):
                continue
            path = get_tick_path(item, trading_date)

            if os.path.exists(path):
                continue
            yield Request(url=self.get_tick_url(trading_date, item['exchange'] + item['code']),
                          meta={'proxy': None,
                                'path': path,
                                'trading_date': trading_date,
                                'item': item},
                          headers=DEFAULT_TICK_HEADER,
                          callback=self.download_tick)

    def start_requests(self):
        item = self.settings.get("security_item")
        trading_dates = self.settings.get("trading_dates")
        if item is not None:
            for request in self.yield_request(item, trading_dates):
                yield request
        else:
            for _, item in get_security_list(start_code=STOCK_START_CODE, end_code=STOCK_END_CODE).iterrows():
                for request in self.yield_request(item):
                    yield request

    def download_tick(self, response):
        content_type_header = response.headers.get('content-type', None)
        if content_type_header.decode("utf-8") == 'application/vnd.ms-excel' or "当天没有数据" in response.body.decode(
                'GB2312'):
            trading_date = response.meta['trading_date']
            security_item = response.meta['item']
            if content_type_header.decode("utf-8") == 'application/vnd.ms-excel':
                content = response.body
            else:
                kdata_df = get_kdata(security_item, the_date=trading_date)
                if trading_date in kdata_df.index:
                    json_data = kdata_df.loc[trading_date, :]
                    content = kdata_to_tick(json_data)
                    self.logger.info(
                        "{} {} generate tick from kdata {}".format(security_item['code'], trading_date, content))
                    content = content.encode('GB2312')

            sina_tick_to_csv(security_item, io.BytesIO(content), trading_date)