def get_timeout():
    return cf.get('time_out') def get_crawl_interal():
    interal = random.randint(cf.get('min_crawl_interal'), cf.get('max_crawl_interal'))
    return interal def get_excp_interal():
    return cf.get('excp_interal') def get_max_retries():
    return cf.get('max_retries') def get_adver_timers():
    return cf.get('adver_timers') @db_commit_decorator
def freeze_account(name, rs):
    """
    :param name: login account
    :param rs: 0 stands for banned，1 stands for normal，2 stands for name or password is invalid
    :return: 
    """
    account = db_session.query(LoginInfo).filter(LoginInfo.name == name).first()
    account.enable = rs
    db_session.commit() class Cookies(object):
    rd_con = redis.StrictRedis(host=redis_args.get('host'), port=redis_args.get('port'),
                               password=redis_args.get('password'), db=redis_args.get('cookies'))

    rd_con_broker = redis.StrictRedis(host=redis_args.get('host'), port=redis_args.get('port'),
                                      password=redis_args.get('password'), db=redis_args.get('broker'))

    @classmethod
    def store_cookies(cls, name, cookies, source, adver_timers):
        pickled_cookies = json.dumps(
            {'cookies': cookies, 'loginTime': datetime.datetime.now().timestamp(),
             'source': source, 'adver_timers': adver_timers + 1})
        cls.rd_con.hset('account', name, pickled_cookies)
        cls.push_in_queue(name)

    @classmethod
    def push_in_queue(cls, name):
        # if the concurrency is large, we can't guarantee there are no reduplicate values
        for i in range(cls.rd_con.llen('account_queue')):
            tn = cls.rd_con.lindex('account_queue', i).decode('utf-8')
            if tn:
                if tn == name:
                    return
        cls.rd_con.rpush('account_queue', name)

    @classmethod
    def fetch_cookies(cls):
        # todo send user a email if it's been blocked
        if mode == 'normal':
            return cls.fetch_cookies_of_normal()

        else:
            return cls.fetch_cookies_of_quick()

    @classmethod
    def fetch_cookies_of_normal(cls):
        # look for available accounts
        for i in range(cls.rd_con.llen('account_queue')):
            name = cls.rd_con.lpop('account_queue').decode('utf-8')
            # during the crawling, some cookies can be banned
            # some account fetched from account_queue can be unavailable
            j_account = cls.rd_con.hget('account', name).decode('utf-8')
            if j_account:
                if cls.check_cookies_timeout(j_account):
                    cls.delete_cookies(name)
                    continue
                cls.rd_con.rpush('account_queue', name)
                account = json.loads(j_account)
                return name, account['cookies'], account['source'], account['adver_timers']
        return None

    @classmethod
    def fetch_cookies_of_quick(cls):
        # record one cookie used by how many host,
        # if the number is bigger than share_host_count，put it to the end of the queue
        # else just fetch and use it
        # todo there are some problems using hostname to mark different hosts because hostname can be the same
        hostname = socket.gethostname()
        my_cookies_name = cls.rd_con.hget('host', hostname)
        if my_cookies_name:
            my_cookies = cls.rd_con.hget('account', my_cookies_name)
            # if cookies is expired, fetch a new one
            if not cls.check_cookies_timeout(my_cookies):
                my_cookies = json.loads(my_cookies.decode('utf-8'))
                return my_cookies_name, my_cookies['cookies'], my_cookies['source'], my_cookies['adver_timers']
            else:
                cls.delete_cookies(my_cookies_name)

        while True:
            try:
                name = cls.rd_con.lpop('account_queue').decode('utf-8')
            except AttributeError:
                return None
            else:
                j_account = cls.rd_con.hget('account', name)

                if cls.check_cookies_timeout(j_account):
                    cls.delete_cookies(name)
                    continue

                j_account = j_account.decode('utf-8')
                # one account maps many hosts（one to many）
                hosts = cls.rd_con.hget('cookies_host', name)
                if not hosts:
                    hosts = dict()
                else:
                    hosts = hosts.decode('utf-8')
                    hosts = json.loads(hosts)
                hosts[hostname] = 1
                cls.rd_con.hset('cookies_host', name, json.dumps(hosts))

                # one host maps one account (one to one)
                account = json.loads(j_account)
                cls.rd_con.hset('host', hostname, name)

                # push the cookie to the head
                if len(hosts) < share_host_count:
                    cls.rd_con.lpush('account_queue', name)
                return name, account['cookies'], account['source'], account['adver_timers']

    @classmethod
    def delete_cookies(cls, name):
        cls.rd_con.hdel('account', name)
        if mode == 'quick':
            cls.rd_con.hdel('cookies_host', name)
        return True

    @classmethod
    def check_login_task(cls):
        if cls.rd_con_broker.llen('login_queue') > 0:
            cls.rd_con_broker.delete('login_queue')

    @classmethod
    def check_cookies_timeout(cls, cookies):
        if cookies is None:
            return True
        if isinstance(cookies, bytes):
            cookies = cookies.decode('utf-8')
        cookies = json.loads(cookies)
        login_time = datetime.datetime.fromtimestamp(cookies['loginTime'])
        if datetime.datetime.now() - login_time > datetime.timedelta(hours=cookie_expire_time):
            crawler.warning('the account has been expired')
            return True
        return False class Urls(object):
    rd_con = redis.StrictRedis(host=redis_args.get('host'), port=redis_args.get('port'),
                               password=redis_args.get('password'), db=redis_args.get('urls'))

    @classmethod
    def store_crawl_url(cls, url, result):
        cls.rd_con.set(url, result) def timeout_decorator(func):
    @wraps(func)
    def time_limit(*args, **kargs):
        try:
            return func(*args, **kargs)
        except Exception as e:
            crawler.error('failed to crawl {url}，here are details:{e}, stack is {stack}'.format(url=args[0], e=e,
                                                                                                stack=format_tb
                                                                                                (e.__traceback__)[0]))
            return ''

    return time_limit def timeout(seconds):
    def crwal_decorator(func):
        def _new_func(oldfunc, result, oldfunc_args, oldfunc_kwargs):
            result.append(oldfunc(*oldfunc_args, **oldfunc_kwargs))

        @wraps(func)
        def wrapper(*args, **kwargs):
            result = []
            # create new args for _new_func, because we want to get the func return val to result list
            new_kwargs = {
                'oldfunc': func,
                'result': result,
                'oldfunc_args': args,
                'oldfunc_kwargs': kwargs
            }

            thd = KThread(target=_new_func, args=(), kwargs=new_kwargs)
            thd.start()
            thd.join(seconds)
            alive = thd.isAlive()
            thd.kill()  # kill the child thread

            if alive:
                try:
                    raise Timeout('request timeout')
                finally:
                    return ''
            else:
                if result:
                    return result[0]
                else:
                    return ''
        return wrapper

    return crwal_decorator @parse_decorator(3)
def is_403(html):
    if "['uid']" not in html and "['nick']" not in html:
        return True

    soup = BeautifulSoup(html, 'html.parser')
    if soup.title:
        if '访问受限' in soup.title.text or '解冻' in soup.title.text:
            return True
        else:
            return False
    else:
        return False @parse_decorator(3)
def is_404(html):
    soup = BeautifulSoup(html, 'html.parser')
    # 前一种情况是处理直接用js实现重定向的页面
    try:
        if "http://weibo.com/sorry?pagenotfound" in html:
            return True
        elif soup.title.text == '404错误':
            return True
        elif html == '':
            return True
        # 处理转发微博的情况
        elif '抱歉，此微博已被作者删除' in html:
            return True
        else:
            return False
    except AttributeError:
        return False def is_complete(html):
    return True if 'uid' in html else False def send_email(from_nick='jtyd_python_spider', to_nick='SpiderUser'):
    msg = gen_msg(warning_info, email_subject, from_nick, to_nick)
    server = smtplib.SMTP(smtp_server, smtp_port)
    server.starttls()
    server.login(from_addr, from_password)
    server.sendmail(from_addr, [to_addr], msg.as_string())
    server.quit()
import os
import signal
import time
import requests
from config.conf import get_timeout, get_crawl_interal, get_excp_interal, get_max_retries, get_adver_timers
from db.login_info import freeze_account
from db.redis_db import Cookies
from db.redis_db import Urls
from decorators.decorator import timeout_decorator, timeout
from logger.log import crawler
from page_parse.basic import is_403, is_404, is_complete
from utils.email_warning import send_email
from utils.headers import headers, personal_message_headers
        return True
    return False


@timeout(200)
@timeout_decorator
def get_page(url, user_verify=True, need_login=True):
    """
    :param url: url to be crawled
    :param user_verify: if it's ajax url, the value is False, else True
    :param need_login: if the url is need to login, the value is True, else False
    :return: return '' if exception happens or status_code != 200
    """
    crawler.info('the crawling url is {url}'.format(url=url))
    count = 0

    while count < max_retries:
        if need_login:
            name_cookies = Cookies.fetch_cookies()

            if name_cookies is None:
                crawler.warning('no cookies in cookies pool, please find out the reason')
                send_email()
                os.kill(os.getppid(), signal.SIGTERM)
        try:
            if need_login:
                resp = requests.get(url, headers=headers, cookies=name_cookies[1], timeout=time_out, verify=False)

                if "$CONFIG['islogin'] = '0'" in resp.text:
                    crawler.warning('account {} has been banned'.format(name_cookies[0]))
                    freeze_account(name_cookies[0], 0)
                    Cookies.delete_cookies(name_cookies[0])
                    continue
            else:
                resp = requests.get(url, headers=headers, timeout=time_out, verify=False)

            page = resp.text

            if page:
                page = page.encode('utf-8', 'ignore').decode('utf-8')
            else:
                continue

            # slow down to aviod being banned
            time.sleep(interal)

            if user_verify:
                if is_banned(resp.url) or is_403(page):
                    crawler.warning('account {} has been banned'.format(name_cookies[0]))
                    freeze_account(name_cookies[0], 0)
                    Cookies.delete_cookies(name_cookies[0])
                    count += 1
                    continue

                if 'verifybmobile' in resp.url:
                    crawler.warning('account {} has been locked，you should use your phone to unlock it'.
                                    format(name_cookies[0]))

                    freeze_account(name_cookies[0], -1)
                    Cookies.delete_cookies(name_cookies[0])
                    continue

                if not is_complete(page):
                    count += 1
                    continue

                if is_404(page):
                    crawler.warning('{url} seems to be 404'.format(url=url))
                    return ''

        except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError, AttributeError) as e:
            crawler.warning('excepitons happens when crawling {}，specific infos are {}'.format(url, e))
            count += 1
            time.sleep(excp_interal)

        else:
            Urls.store_crawl_url(url, 1)
            return page

    crawler.warning('max tries for {}，check the url in redis db2'.format(url))
    Urls.store_crawl_url(url, 0)
    return ''

@timeout(200)
@timeout_decorator
def send_personal_message(target_uid, adver_message, user_verify=True, need_login=True):
    """
    :param url: url to be crawled
    :param user_verify: if it's ajax url, the value is False, else True
    :param need_login: if the url is need to login, the value is True, else False
    :return: return '' if exception happens or status_code != 200
    """
    crawler.info('the send_personal_message uid is {uid}'.format(uid=str(target_uid)))
    count = 0

    while count < max_retries:
        if need_login:
            name_cookies = Cookies.fetch_cookies()
            print(name_cookies)
            # check adver_timers
            if int(name_cookies[3]) >= int(adver_timers):
                continue

            if name_cookies is None:
                crawler.warning('no cookies in cookies pool, please find out the reason')
                send_email()
                os.kill(os.getppid(), signal.SIGTERM)
        try:
            if need_login:
                resp = requests.post('http://api.weibo.com/webim/2/direct_messages/new.json?source='+str(name_cookies[2]),
                              data={'text': adver_message, 'uid':str(target_uid)},
                              cookies=name_cookies[1], headers=personal_message_headers)

                if "error" in resp.text:
                    crawler.warning('account {} has been banned, resp.text is: {}'.format(name_cookies[0], resp.text))
                    freeze_account(name_cookies[0], 0)
                    Cookies.delete_cookies(name_cookies[0])
                    continue
                else:
                    # update adver_times

                    Cookies.store_cookies(name_cookies[0], name_cookies[1], name_cookies[2], 1)