def get_security_list(security_type='stock', exchanges=None, start_code=None, end_code=None,
                      mode='simple', start_list_date=None, codes=None):
    """
    get security list.

    Parameters
    ----------
    security_type : str
        {‘stock’, 'future'},default: stock
    exchanges : str or list
        ['sh', 'sz','nasdaq','nyse','amex','shfe','dce','zce'],default: ['sh','sz']
    start_code : str
        the start code,work with end,default:None
        if using codes,it would be ignored
    end_code : str
        the end code,works with start,default:None
        if using codes,it would be ignored
    mode : str
        whether parse more security info,{'simple','es'},default:'simple'
    start_list_date : Timestamp str or Timestamp
        the filter for start list date,default:None
    codes : list
        the exact codes to query,default:None

    Returns
    -------
    DataFrame
        the security list

    """
    df = pd.DataFrame()
    if type(exchanges) == str:
        exchanges = [exchanges]

    if not exchanges:
        exchanges = SECURITY_TYPE_MAP_EXCHANGES[security_type]

    if security_type == 'index':
        df = df.append(pd.DataFrame(CHINA_STOCK_SH_INDEX), ignore_index=True)
        df = df.append(pd.DataFrame(CHINA_STOCK_SZ_INDEX), ignore_index=True)
        df = df.append(pd.DataFrame(USA_STOCK_NASDAQ_INDEX), ignore_index=True)
    else:
        for exchange in exchanges:
            the_path = get_security_list_path(security_type, exchange)
            if os.path.exists(the_path):
                converters = None

                if mode == 'es' and security_type == 'stock':
                    converters = {'code': str,
                                  'sinaIndustry': convert_to_list_if_need,
                                  'sinaConcept': convert_to_list_if_need,
                                  'sinaArea': convert_to_list_if_need}
                if converters:
                    df = df.append(pd.read_csv(the_path, converters=converters), ignore_index=True)
                else:
                    df = df.append(pd.read_csv(the_path, dtype=str), ignore_index=True)

    if not df.empty > 0:
        df = df_for_date_range(df, start_date=start_list_date)

        df = df.set_index(df['code'], drop=False)
        df = df.sort_index()

        if codes:
            df = df[df["code"].isin(codes)]
        elif start_code and end_code:
            df = df[(df["code"] >= start_code) & (df["code"] <= end_code)]

        if security_type != 'cryptocurrency':
            df = df.drop_duplicates(subset='code', keep='last')

    return df KDATA_STOCK_COL = ['timestamp', 'code', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
                   'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap', 'factor'] KDATA_COLUMN_163 = ['timestamp', 'code', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
                    'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap'] KDATA_INDEX_COLUMN_163 = ['timestamp', 'code', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
                          'preClose', 'change', 'changePct'] KDATA_INDEX_COL = ['timestamp', 'code', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
                   'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap', 'pe'] def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
    source = adjust_source(item, source)
    if source == 'sina':
        if not year and not quarter:
            return os.path.join(get_kdata_dir(item, fuquan), 'dayk.csv')
        else:
            return os.path.join(get_kdata_dir(item, fuquan), '{}Q{}.csv'.format(year, quarter))
    else:
        return os.path.join(get_kdata_dir(item, fuquan), '{}_dayk.csv'.format(source)) STOCK_START_CODE = '000001' STOCK_END_CODE = '666666' def random_proxy(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        for request in func(*args, **kwargs):
            proxy_df = get_checked_proxy()
            if proxy_df is not None:
                request._meta['proxy'] = proxy_df.at[random.choice(proxy_df.index), 'url']
            yield request

    return wrapper def get_security_id(security_type, exchange, code):
def init_process_log(file_name, log_dir=None):
def chrome_copy_header_to_dict(src):
def generate_csv_line(*items):
def gen_security_id(type, exchange, code):
def kdata_to_tick(kdata_json):
def get_tick_item(path, the_date, security_item):
def get_datetime(str):
def get_year_quarter(time):
def get_quarters(start, end=datetime.date.today()):
def fill_doc_type(doc_type, json_object):
def to_float(str, default=None):
def get_exchange(code):
def direction_to_int(direction):
def read_csv(f, encoding, sep=None, na_values=None):
def sina_tick_to_csv(security_item, the_content, the_date):
def get_file_name(the_path):
def index_df_with_time(df, index='timestamp'):
def is_same_date(one, two):
def is_same_time(one, two):
def get_report_period(the_date=datetime.datetime.today().date()):
def to_timestamp(the_time):
def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
def to_epoch_millis(the_time):
def next_date(the_time):
def drop_duplicate(the_list):
import io
import os
import pandas as pd
import scrapy
from datetime import datetime
from scrapy import Request
from scrapy import signals
from fooltrader.api.technical import get_security_list
from fooltrader.contract.data_contract import KDATA_STOCK_COL, KDATA_COLUMN_163, KDATA_INDEX_COLUMN_163, \
    KDATA_INDEX_COL
from fooltrader.contract.files_contract import get_kdata_path
from fooltrader.settings import STOCK_START_CODE, STOCK_END_CODE
from fooltrader.spiders.common import random_proxy
from fooltrader.utils import utils
# -*- coding: utf-8 -*-





class StockKdata163Spider(scrapy.Spider):
    name = "stock_kdata_163"

    custom_settings = {
        # 'DOWNLOAD_DELAY': 2,
        # 'CONCURRENT_REQUESTS_PER_DOMAIN': 8,

        'SPIDER_MIDDLEWARES': {
            'fooltrader.middlewares.FoolErrorMiddleware': 1000,
        }
    }

    # 指定日期的话，是用来抓增量数据的
    # 如果需要代理请打开
    # @random_proxy
    def yield_request(self, item, start_date=None, end_date=None):
        data_path = get_kdata_path(item, source='163')

        if start_date:
            start = start_date.strftime('%Y%m%d')
        else:
            start = item['listDate'].replace('-', '')

        if end_date:
            end = end_date.strftime('%Y%m%d')
        else:
            end = datetime.today().strftime('%Y%m%d')

        if not os.path.exists(data_path) or start_date or end_date:
            if item['exchange'] == 'sh':
                exchange_flag = 0
            else:
                exchange_flag = 1
            url = self.get_k_data_url(exchange_flag, item['code'], start, end)
            yield Request(url=url, meta={'path': data_path, 'item': item},
                          callback=self.download_day_k_data)

    def start_requests(self):
        item = self.settings.get("security_item")
        start_date = self.settings.get("start_date")
        end_date = self.settings.get("end_date")
        if item is not None:
            for request in self.yield_request(item, start_date, end_date):
                yield request
        else:
            for _, item in get_security_list(start_code=STOCK_START_CODE, end_code=STOCK_END_CODE).iterrows():
                for request in self.yield_request(item):
                    yield request

    def download_day_k_data(self, response):
        path = response.meta['path']
        item = response.meta['item']

        try:
            # 已经保存的csv数据
            if os.path.exists(path):
                saved_df = pd.read_csv(path, dtype=str)
            else:
                saved_df = pd.DataFrame()

            df = utils.read_csv(io.BytesIO(response.body), encoding='GB2312', na_values='None')
            df['code'] = item['code']
            df['securityId'] = item['id']
            df['name'] = item['name']
            # 指数数据
            if item['type'] == 'index':
                df = df.loc[:,
                     ['日期', 'code', 'name', '最低价', '开盘价', '收盘价', '最高价', '成交量', '成交金额', 'securityId', '前收盘', '涨跌额',
                      '涨跌幅']]
                df['turnoverRate'] = None
                df['tCap'] = None
                df['mCap'] = None
                df['pe'] = None

                df.columns = KDATA_INDEX_COL