<gh_stars>1-10
from dbModels import ProxyModelDispatcher
from proxy import PROXIES, FREE_PROXIES
from agents import AGENTS_ALL
import logging as log

import random


class CustomHttpProxyFromMysqlMiddleware(object):
    proxies = FREE_PROXIES

    def process_request(self, request, spider):
        # TODO implement complex proxy providing algorithm
        if self.use_proxy(request):
            p = random.choice(self.proxies)
            try:
                request.meta['proxy'] = "http://%s" % p['ip_port']
                print(request.meta['proxy'])
            except Exception, e:
                # log.msg("Exception %s" % e, _level=log.CRITICAL)
                log.critical("Exception %s" % e)

    def use_proxy(self, request):
        """
        using direct download for depth <= 2
        using proxy with probability 0.3
        """
        # if "depth" in request.meta and int(request.meta['depth']) <= 2:
        #    return False
        # i = random.randint(1, 10)
        # return i <= 2
        return True


class CustomHttpProxyMiddleware(object):
    def process_request(self, request, spider):
        # TODO implement complex proxy providing algorithm
        if self.use_proxy(request):
            p = random.choice(PROXIES)
            try:
                request.meta['proxy'] = "http://%s" % p['ip_port']
            except Exception, e:
                # log.msg("Exception %s" % e, _level=log.CRITICAL)
                log.critical("Exception %s" % e)

    def use_proxy(self, request):
        """
        using direct download for depth <= 2
        using proxy with probability 0.3
        """
        # if "depth" in request.meta and int(request.meta['depth']) <= 2:
        #    return False
        # i = random.randint(1, 10)
        # return i <= 2
        return True


class CustomUserAgentMiddleware(object):
    def process_request(self, request, spider):
        agent = random.choice(AGENTS_ALL)
        if self.use_proxy(request):
            proxyModel = ProxyModelDispatcher.getProxyInfo()
            proxyList = None
            p = None
            if proxyModel is not None:
                proxyList = proxyModel['proxyList']
            if proxyList is not None and len(proxyList) > 0:
                p = random.choice(proxyList)
            # p = [
            #     {"ip_port": "172.16.58.3:8080"},  # tor via privoxy
            # ]
            try:
                request.meta['proxy'] = "http://%s" % p['HTTP']
            except Exception, e:
                pass
        request.headers['User-Agent'] = agent

    def use_proxy(self, request):
        """
        using direct download for depth <= 2
        using proxy with probability 0.3
        """
        # if "depth" in request.meta and int(request.meta['depth']) <= 2:
        #    return False
        # i = random.randint(1, 10)
        # return i <= 2
        return False

from dbModels import ProxyModelDispatcher
from proxy import PROXIES, FREE_PROXIES
from agents import AGENTS_ALL
import logging as log

import random


class CustomHttpProxyFromMysqlMiddleware(object):
    proxies = FREE_PROXIES

    def process_request(self, request, spider):
        # TODO implement complex proxy providing algorithm
        if self.use_proxy(request):
            p = random.choice(self.proxies)
            try:
                request.meta['proxy'] = "http://%s" % p['ip_port']
                print(request.meta['proxy'])
            except Exception, e:
                # log.msg("Exception %s" % e, _level=log.CRITICAL)
                log.critical("Exception %s" % e)

    def use_proxy(self, request):
        """
        using direct download for depth <= 2
        using proxy with probability 0.3
        """
        # if "depth" in request.meta and int(request.meta['depth']) <= 2:
        #    return False
        # i = random.randint(1, 10)
        # return i <= 2
        return True


class CustomHttpProxyMiddleware(object):
    def process_request(self, request, spider):
        # TODO implement complex proxy providing algorithm
        if self.use_proxy(request):
            p = random.choice(PROXIES)
            try:
                request.meta['proxy'] = "http://%s" % p['ip_port']
            except Exception, e:
                # log.msg("Exception %s" % e, _level=log.CRITICAL)
                log.critical("Exception %s" % e)

    def use_proxy(self, request):
        """
        using direct download for depth <= 2
        using proxy with probability 0.3
        """
        # if "depth" in request.meta and int(request.meta['depth']) <= 2:
        #    return False
        # i = random.randint(1, 10)
        # return i <= 2
        return True


class CustomUserAgentMiddleware(object):
    def process_request(self, request, spider):
        agent = random.choice(AGENTS_ALL)
        if self.use_proxy(request):
            proxyModel = ProxyModelDispatcher.getProxyInfo()
            proxyList = None
            p = None
            if proxyModel is not None:
                proxyList = proxyModel['proxyList']
            if proxyList is not None and len(proxyList) > 0:
                p = random.choice(proxyList)
            # p = [
            #     {"ip_port": "172.16.58.3:8080"},  # tor via privoxy
            # ]
            try:
                request.meta['proxy'] = "http://%s" % p['HTTP']
            except Exception, e:
                pass
        request.headers['User-Agent'] = agent

    def use_proxy(self, request):
        """
        using direct download for depth <= 2
        using proxy with probability 0.3
        """
        # if "depth" in request.meta and int(request.meta['depth']) <= 2:
        #    return False
        # i = random.randint(1, 10)
        # return i <= 2
        return False
# TODO implement complex proxy providing algorithm # log.msg("Exception %s" % e, _level=log.CRITICAL) using direct download for depth <= 2         using proxy with probability 0.3 # if "depth" in request.meta and int(request.meta['depth']) <= 2: #    return False # i = random.randint(1, 10) # return i <= 2 # TODO implement complex proxy providing algorithm # log.msg("Exception %s" % e, _level=log.CRITICAL) using direct download for depth <= 2         using proxy with probability 0.3 # if "depth" in request.meta and int(request.meta['depth']) <= 2: #    return False # i = random.randint(1, 10) # return i <= 2 # p = [ #     {"ip_port": "172.16.58.3:8080"},  # tor via privoxy # ] using direct download for depth <= 2         using proxy with probability 0.3 # if "depth" in request.meta and int(request.meta['depth']) <= 2: #    return False # i = random.randint(1, 10) # return i <= 2