@urlmatch(netloc=r'(.*\.)?free-proxy-list\.net$')
def free_proxy_mock(url, request):
    return """<table border="0" cellpadding="0" cellspacing="0" id="proxylisttable"
id="proxylisttable">\n
<thead>\n
<tr>\n
    <th>IP Address</th>
    \n
    <th>Port</th>
    \n
    <th>Code</th>
    \n
    <th>Country</th>
    \n
    <th>Anonymity</th>
    \n
    <th>Google</th>
    \n
    <th>Https</th>
    \n
    <th>Last Checked</th>
    \n
</tr>
\n
</thead>
\n
<tbody>
<tr>
    <td>138.197.136.46</td>
    <td>3128</td>
    <td>CA</td>
    <td>Canada</td>
    <td>anonymous</td>
    <td>no</td>
    <td>no</td>
    <td>7 seconds ago</td>
</tr>
\n
<tr>
    <td>177.207.75.227</td>
    <td>8080</td>
    <td>BR</td>
    <td>Brazil</td>
    <td>transparent</td>
    <td>no</td>
    <td>no</td>
    <td>2 hours 21 minutes ago</td>
</tr>
\n
</tbody>
\n
<tfoot>\n
<tr>\n
    <th class="input"><input type="text"/></th>
    \n
    <th></th>
    \n
    <th></th>
    \n
    <th></th>
    \n
    <th></th>
    \n
    <th></th>
    \n
    <th></th>
    \n
    <th></th>
    \n
</tr>
\n
</tfoot>
\n
</table>""" @urlmatch(netloc=r'(.*\.)?proxyfor\.eu')
def proxy_for_eu_mock(url, request):
    return """<table class="proxy_list">
    <tr>
        <th>IP</th>
        <th>Port</th>
        <th>Country</th>
        <th>Anon</th>
        <th>Speed</th>
        <th> Check</th>
        <th>Cookie/POST</th>
    </tr>
    <tr>
        <td>107.151.136.222</td>
        <td>80</td>
        <td>United States</td>
        <td>HIGH</td>
        <td>1.643</td>
        <td>2016-04-12 17:02:43</td>
        <td>Yes/Yes</td>
    </tr>
    <tr>
        <td>37.187.253.39</td>
        <td>8115</td>
        <td>France</td>
        <td>HIGH</td>
        <td>12.779</td>
        <td>2016-04-12 14:36:18</td>
        <td>Yes/Yes</td>
    </tr>
</table>""" @urlmatch(netloc=r'(.*\.)?rebro\.weebly\.com$')
def rebro_weebly_mock(url, request):
    return """<div class="paragraph" style="text-align:left;"><strong><font color="#3ab890" size="3"><font
        color="#d5d5d5">IP:Port</font></font></strong><br/><font
        size="2"><strong><font color="#33a27f">213.149.105.12:8080<br/>119.188.46.42:8080</font></strong></font><br/><span></span>
</div>


<div class="paragraph" style="text-align:left;"><font size="2"><strong><font size="3"><font color="#3ab890">Country</font></font></strong></font><font size="2">
    <br />Montenegro<br />China<br /></font><br /><span></span>
</div>

<div class="paragraph" style="text-align:left;"><font size="2"><strong><font color="#3ab890" size="3">Status</font></strong></font><br /><font size="2">
    Elite &amp; Anonymous<br />Elite &amp; Anonymous<br /></font><br /><span></span>
</div>

""" @urlmatch(netloc=r'(.*\.)?www\.premproxy\.com')
def prem_mock(url, request):
    return """
    <head>
        <script src="/js/test.js"></script>
    </head>
    <div id="proxylist">\n
    <tr class="anon">\n
        <th><a href="/list/ip-address-01.htm" title="Proxy List sorted by ip address">IP address</a></th>
        \n
        <th><a href="/list/" title="Proxy List sorted by anonymity level">Anonymity</a></th>
        \n
        <th><a href="/list/time-01.htm" title="Proxy List sorted by updated time">Checked</a></th>
        \n
        <th><a href="/list/type-01.htm" title="Proxy list sorted by country">Country</a></th>
        \n
        <th><dfn title="City or State\\Region ">City</dfn></th>
        \n
        <th><dfn title="Internet Service Provider">ISP</dfn></th>
        \n
    </tr>
    \n
    <div id="navbar">
        <ul class="pagination"><li class="active"><a href="/list/">1</a></li><li><a href="02.htm">2</a></li></ul>
    </div>
    \n
    <tr class="anon">
        <td data-label="IP:port "><span><input type="checkbox" name="proxyIp[]" value="191.252.61.28|r60e6"></span>191.252.61.28:<span class="r60e6"></span></td>
        <td data-label="Anonymity Type: ">high-anonymous</td>
        <td data-label="Checked: ">Apr-18, 17:18</td>
        <td data-label="Country: ">Brazil</td>
        <td data-label="City: ">S\xe3o Jos\xe9 Dos Campos</td>
        <td data-label="ISP: "><dfn title="Locaweb Servi\xe7os de Internet S/A">Locaweb
            Servi\xe7o...</dfn></td>
    </tr>
    \n
    <tr class="anon">
        <td data-label="IP:port "><span><input type="checkbox" name="proxyIp[]" value="167.114.203.141|r63c5"></span>167.114.203.141:<span class="r63c5"></span></td>
        <td data-label="Anonymity Type: ">transparent</td>
        <td data-label="Checked: ">Apr-18, 13:22</td>
        <td data-label="Country: ">Canada</td>
        <td data-label="City: ">Montr\xe9al (QC)</td>
        <td data-label="ISP: ">OVH Hosting</td>
    </tr>
    \n
    <tr class="anon">
        <td data-label="IP:port "><span><input type="checkbox" name="proxyIp[]" value="152.251.141.93|r63c5"></span>152.251.141.93:<span class="r63c5"></span></td>
        <td data-label="Anonymity Type: ">elite </td>
        <td data-label="Checked: ">Jul-16, 04:39</td>
        <td data-label="Country: ">Brazil</td>
        <td data-label="City: ">&nbsp;</td>
        <td data-label="ISP: ">Vivo</td>
    </tr>
    \n
    <tr><td colspan="6"><span><input type="checkbox" name="" value="" onclick="checkAll(this)"></span>Select All Proxies</td></tr>
</div>""" @urlmatch(netloc=r'(.*\.)?sslproxies\.org$')
def sslproxy_mock(url, request):
    return """<table class="table table-striped table-bordered" cellspacing="0" width="100%" id="proxylisttable">
    <thead>
    <tr>
        <th>IP Address</th>
        <th>Port</th>
        <th>Code</th>
        <th class='hm'>Country</th>
        <th>Anonymity</th>
        <th class='hm'>Google</th>
        <th class='hx'>Https</th>
        <th class='hm'>Last Checked</th>
    </tr>
    </thead>
    <tbody>
    <tr>
        <td>24.211.89.146</td>
        <td>8080</td>
        <td>US</td>
        <td class='hm'>United States</td>
        <td>elite proxy</td>
        <td class='hm'>no</td>
        <td class='hx'>yes</td>
        <td class='hm'>8 seconds ago</td>
    </tr>
    <tr>
        <td>187.84.222.153</td>
        <td>80</td>
        <td>BR</td>
        <td class='hm'>Brazil</td>
        <td>anonymous</td>
        <td class='hm'>no</td>
        <td class='hx'>yes</td>
        <td class='hm'>1 minute ago</td>
    </tr>
    <tr>
        <td>41.193.238.249</td>
        <td>8080</td>
        <td>ZA</td>
        <td class='hm'>South Africa</td>
        <td>elite proxy</td>
        <td class='hm'>no</td>
        <td class='hx'>yes</td>
        <td class='hm'>1 minute ago</td>
    </tr>
    </tbody>
    <tfoot>
        <tr>
        <th class="input"><input type="text" /></th>
        <th></th><th></th>
        <th class='hm'></th>
        <th></th>
        <th class='hm'></th>
        <th class='hx'></th>
        <th class='hm'></th>
        </tr>
    </tfoot>
</table>
        """ def sslproxy_mock(url, request):
def free_proxy_mock(url, request):
def proxy_for_eu_mock(url, request):
def rebro_weebly_mock(url, request):
def prem_mock(url, request):
def prem_js_mock(url, request): class FreeProxyParser(UrlParser):
    def __init__(self, id, web_url, timeout=None):
        UrlParser.__init__(self, id=id, web_url=web_url, timeout=timeout)

    def parse_proxyList(self):
        curr_proxy_list = []
        try:
            response = requests.get(self.get_url(), timeout=self.timeout)
            if not response.ok:
                logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
                return []

            content = response.content
            soup = BeautifulSoup(content, "html.parser")
            table = soup.find("table", attrs={"id": "proxylisttable"})

            # The first tr contains the field names.
            headings = [th.get_text() for th in table.find("tr").find_all("th")]

            datasets = []
            for row in table.find_all("tr")[1:-1]:
                dataset = zip(headings, (td.get_text() for td in row.find_all("td")))
                if dataset:
                    datasets.append(dataset)

            for dataset in datasets:
                proxy_obj = self.create_proxy_object(dataset)
                # Make sure it is a Valid Proxy Address
                if proxy_obj is not None and UrlParser.valid_ip_port(proxy_obj.get_address()):
                    curr_proxy_list.append(proxy_obj)
                else:
                    logger.debug("Proxy Invalid: {}".format(dataset))
        except AttributeError as e:
            logger.error("Provider {0} failed with Attribute error: {1}".format(self.id, e))
        except KeyError as e:
            logger.error("Provider {0} failed with Key error: {1}".format(self.id, e))
        except Exception as e:
            logger.error("Provider {0} failed with Unknown error: {1}".format(self.id, e))
        finally:
            return curr_proxy_list

    def create_proxy_object(self, dataset):
        # Check Field[0] for tags and field[1] for values!
        ip = ""
        port = None
        anonymity = AnonymityLevel.UNKNOWN
        country = None
        protocols = []
        for field in dataset:
            if field[0] == 'IP Address':
                # Make sure it is a Valid IP
                ip = field[1].strip()  # String strip()
                # Make sure it is a Valid IP
                if not UrlParser.valid_ip(ip):
                    logger.debug("IP with Invalid format: {}".format(ip))
                    return None
            elif field[0] == 'Port':
                port = field[1].strip()  # String strip()
            elif field[0] == 'Anonymity':
                anonymity = AnonymityLevel.get(field[1].strip())  # String strip()
            elif field[0] == 'Country':
                country = field[1].strip()  # String strip()
            elif field[0] == 'Https':
                if field[1].strip().lower() == 'yes': protocols.extend([Protocol.HTTP, Protocol.HTTPS])
                elif field[1].strip().lower() == 'no': protocols.append(Protocol.HTTP)
        return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country, protocols=protocols)

    def __str__(self):
        return "{0} parser of '{1}' with required bandwidth: '{2}' KBs" \
            .format(self.id, self.url, self.minimum_bandwidth_in_KBs) class ProxyForEuParser(UrlParser):
    def __init__(self, id, web_url, bandwidth=None, timeout=None):
        UrlParser.__init__(self, id=id, web_url=web_url, bandwidth_KBs=bandwidth, timeout=timeout)

    def parse_proxyList(self):
        curr_proxy_list = []
        try:
            response = requests.get(self.get_url(), timeout=self.timeout)

            if not response.ok:
                logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
                return []

            content = response.content
            soup = BeautifulSoup(content, "html.parser")
            table = soup.find("table", attrs={"class": "proxy_list"})

            # The first tr contains the field names.
            headings = [th.get_text() for th in table.find("tr").find_all("th")]

            datasets = []
            for row in table.find_all("tr")[1:]:
                dataset = zip(headings, (td.get_text() for td in row.find_all("td")))
                datasets.append(dataset)

            for dataset in datasets:
                # Avoid Straggler proxies and make sure it is a Valid Proxy Address
                proxy_obj = self.create_proxy_object(dataset)
                if proxy_obj is not None and UrlParser.valid_ip_port(proxy_obj.get_address()):
                    curr_proxy_list.append(proxy_obj)
                else:
                    logger.debug("Proxy Invalid: {}".format(dataset))
        except AttributeError as e:
            logger.error("Provider {0} failed with Attribute error: {1}".format(self.id, e))
        except KeyError as e:
            logger.error("Provider {0} failed with Key error: {1}".format(self.id, e))
        except Exception as e:
            logger.error("Provider {0} failed with Unknown error: {1}".format(self.id, e))
        finally:
            return curr_proxy_list

    def create_proxy_object(self, dataset):
        ip = ""
        port = None
        anonymity = AnonymityLevel.UNKNOWN
        country = None
        # Check Field[0] for tags and field[1] for values!
        for field in dataset:
            # Discard slow proxies! Speed is in KB/s
            if field[0] == 'Speed':
                if float(field[1]) < self.get_min_bandwidth():
                    logger.debug("Proxy with low bandwidth: {}".format(float(field[1])))
                    return None
            if field[0] == 'IP':
                ip = field[1].strip()  # String strip()
                # Make sure it is a Valid IP
                if not UrlParser.valid_ip(ip):
                    logger.debug("IP with Invalid format: {}".format(ip))
                    return None
            elif field[0] == 'Port':
                port = field[1].strip()  # String strip()
            elif field[0] == 'Anon':
                anonymity = AnonymityLevel.get(field[1].strip())  # String strip()
            elif field[0] == 'Country':
                country = field[1].strip()  # String strip()
        return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country)

    def __str__(self):
        return "ProxyForEU Parser of '{0}' with required bandwidth: '{1}' KBs" \
            .format(self.url, self.minimum_bandwidth_in_KBs) class RebroWeeblyParser(UrlParser):
    def __init__(self, id, web_url, timeout=None):
        self.top_proxy_path = "proxy-list.html"
        self.txt_proxy_path = "txt-lists.html"
        UrlParser.__init__(self, id=id, web_url=web_url, timeout=timeout)

    def parse_proxyList(self, use_top15k=False):
        curr_proxy_list = []
        try:
            response = requests.get(self.get_url() + "/" + self.top_proxy_path, timeout=self.timeout)

            if not response.ok:
                logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
                return []

            content = response.content
            soup = BeautifulSoup(content, "html.parser")
            all_divs = soup.findAll("div", attrs={"class": "paragraph", 'style': "text-align:left;"})
            # address_table = soup.find("div", attrs={"class": "paragraph", 'style': "text-align:left;"})
            # .find('font', attrs={'color': '#33a27f'})
            # Parse Top Proxy List page
            address_list = []
            country_list = []
            anonymity_list = []
            for div in all_divs:
                address_div = div.find('font', attrs={'color': '#33a27f'})
                if address_div is not None:
                    for row in [x for x in address_div.contents if getattr(x, 'name', None) != 'br']:
                        address_list.append(str(row))
                curr_div = div.findAll('font', attrs={'size': '2'})
                if curr_div[0] is not None:
                    row_data = []
                    # font -> strong -> font
                    title = curr_div[0].contents[0].contents[0].contents[0]
                    for row in [x for x in curr_div[-1].contents if getattr(x, 'name', None) != 'br']:
                        row_data.append(str(row))
                    if 'Country' in str(title):
                        country_list.extend(row_data)
                    if 'Status' in str(title):
                        anonymity_list.extend(row_data)
            for address, country, anonymity in zip(address_list, country_list, anonymity_list):
                # Make sure it is a Valid Proxy Address
                proxy_obj = self.create_proxy_object(address, country, anonymity)
                if proxy_obj is not None and UrlParser.valid_ip_port(proxy_obj.get_address()):
                    curr_proxy_list.append(proxy_obj)
                else:
                    logger.debug("Proxy Invalid: {}".format(row))
            # Usually these proxies are stale
            if use_top15k:
                # Parse 15k Nodes Text file (named *-all-*.txt)
                content = requests.get(self.get_url() + "/" + self.txt_proxy_path).content
                soup = BeautifulSoup(content, "html.parser")
                table = soup.find("div", attrs={"class": "wsite-multicol-table-wrap"})
                for link in table.findAll('a'):
                    current_link = link.get('href')
                    if current_link is not None and "all" in current_link:
                        self.txt_proxy_path = current_link
                more_content = requests.get(self.get_url() + self.txt_proxy_path).text
                for proxy_address in more_content.split():
                    if UrlParser.valid_ip_port(proxy_address):
                        proxy_obj = self.create_proxy_object(row)
                        curr_proxy_list.append(proxy_obj)
        except AttributeError as e:
            logger.error("Provider {0} failed with Attribute error: {1}".format(self.id, e))
        except KeyError as e:
            logger.error("Provider {0} failed with Key error: {1}".format(self.id, e))
        except Exception as e:
            logger.error("Provider {0} failed with Unknown error: {1}".format(self.id, e))
        finally:
            return curr_proxy_list

    def create_proxy_object(self, address, country, anonymity):
        # Make sure it is a Valid IP
        ip = address.strip().split(":")[0]
        if not UrlParser.valid_ip(ip):
            logger.debug("IP with Invalid format: {}".format(ip))
            return None
        port = address.strip().split(":")[1]
        country = country.strip()
        anonymity = AnonymityLevel.get(anonymity.strip())

        return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country)

    def __str__(self):
        return "RebroWeebly Parser of '{0}' with required bandwidth: '{1}' KBs" \
            .format(self.url, self.minimum_bandwidth_in_KBs) class PremProxyParser(UrlParser):
    def __init__(self, id, web_url, timeout=None):
        self.base_url = web_url
        web_url += "/list/"
        # Ports decoded by the JS unpacker
        self.js_unpacker = None
        UrlParser.__init__(self, id=id, web_url=web_url, timeout=timeout)

    def parse_proxyList(self):
        curr_proxy_list = []
        try:
            # Parse all proxy pages -> format: /list/{num}.htm
            # Get the pageRange from the 'pagination' table
            page_set = self.get_pagination_set()
            logger.debug("Pages: {}".format(page_set))
            # One JS unpacker per provider (not per page)
            self.js_unpacker = self.init_js_unpacker()

            for page in page_set:
                response = requests.get("{0}{1}".format(self.get_url(), page), timeout=self.timeout)
                if not response.ok:
                    # Could not parse ANY page - Let user know
                    if not curr_proxy_list:
                        logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
                    # Return proxies parsed so far
                    return curr_proxy_list
                content = response.content
                soup = BeautifulSoup(content, "html.parser", from_encoding="iso-8859-1")

                table = soup.find("div", attrs={"id": "proxylist"})
                # The first tr contains the field names.
                headings = [th.get_text() for th in table.find("tr").find_all("th")]
                # skip last 'Select All' row
                for row in table.find_all("tr")[1:-1]:
                    td_row = row.find("td")
                    portKey = td_row.find('span', attrs={'class': True}).get('class')[0]
                    port = self.js_unpacker.get_port(portKey)
                    proxy_obj = self.create_proxy_object(row, port)
                    # Make sure it is a Valid Proxy Address
                    if proxy_obj is not None and UrlParser.valid_ip(proxy_obj.ip) and UrlParser.valid_port(port):
                        curr_proxy_list.append(proxy_obj)
                    else:
                        logger.debug("Proxy Invalid: {}".format(proxy_obj.to_str()))
        except AttributeError as e:
            logger.error("Provider {0} failed with Attribute error: {1}".format(self.id, e))
        except KeyError as e:
            logger.error("Provider {0} failed with Key error: {1}".format(self.id, e))
        except Exception as e:
            logger.error("Provider {0} failed with Unknown error: {1}".format(self.id, e))
        finally:
            return curr_proxy_list

    def get_pagination_set(self):
        response = requests.get(self.get_url(), timeout=self.timeout)
        page_set = set()
        # Could not parse pagination page - Let user know
        if not response.ok:
            logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
            return page_set
        content = response.content
        soup = BeautifulSoup(content, "html.parser")
        for ultag in soup.find_all('ul', {'class': 'pagination'}):
            for litag in ultag.find_all('li'):
                page_ref = litag.a.get('href')
                # Skip current page '/list'
                if page_ref.endswith(('htm', 'html')):
                    page_set.add(page_ref)
                else:
                    page_set.add("")
        return page_set

    def init_js_unpacker(self):
        response = requests.get(self.get_url(), timeout=self.timeout)
        # Could not parse provider page - Let user know
        if not response.ok:
            logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
            return None
        content = response.content
        soup = BeautifulSoup(content, "html.parser")

        # js file contains the values for the ports
        for script in soup.findAll('script'):
            if '/js/' in script.get('src'):
                jsUrl = self.base_url + script.get('src')
                return JsUnPacker(jsUrl)
        return None

    def create_proxy_object(self, row, port):
        for td_row in row.findAll("td"):
            if td_row.attrs['data-label'] == 'IP:port ':
                text = td_row.text.strip()
                ip = text.split(":")[0]
                # Make sure it is a Valid IP
                if not UrlParser.valid_ip(ip):
                    logger.debug("IP with Invalid format: {}".format(ip))
                    return None
            elif td_row.attrs['data-label'] == 'Anonymity Type: ':
                anonymity = AnonymityLevel.get(td_row.text.strip())
            elif td_row.attrs['data-label'] == 'Country: ':
                country = td_row.text.strip()
            protocols = [Protocol.HTTP]
        return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country, protocols=protocols)

    def __str__(self):
        return "{0} parser of '{1}' with required bandwidth: '{2}' KBs" \
            .format(self.id, self.url, self.minimum_bandwidth_in_KBs) class SslProxyParser(UrlParser):
    def __init__(self, id, web_url, timeout=None):
        UrlParser.__init__(self, id=id, web_url=web_url, timeout=timeout)

    def parse_proxyList(self):
        curr_proxy_list = []
        try:
            response = requests.get(self.get_url(), timeout=self.timeout)
            if not response.ok:
                logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
                return []

            content = response.content
            soup = BeautifulSoup(content, "html.parser")
            table = soup.find("table", attrs={"id": "proxylisttable"})

            # The first tr contains the field names.
            headings = [th.get_text() for th in table.find("tr").find_all("th")]

            datasets = []
            for row in table.find_all("tr")[1:-1]:
                dataset = zip(headings, (td.get_text() for td in row.find_all("td")))
                if dataset:
                    datasets.append(dataset)

            for dataset in datasets:
                proxy_obj = self.create_proxy_object(dataset)
                # Make sure it is a Valid Proxy Address
                if proxy_obj is not None and UrlParser.valid_ip_port(proxy_obj.get_address()):
                    curr_proxy_list.append(proxy_obj)
                else:
                    logger.debug("Proxy Invalid: {}".format(dataset))
        except AttributeError as e:
            logger.error("Provider {0} failed with Attribute error: {1}".format(self.id, e))
        except KeyError as e:
            logger.error("Provider {0} failed with Key error: {1}".format(self.id, e))
        except Exception as e:
            logger.error("Provider {0} failed with Unknown error: {1}".format(self.id, e))
        finally:
            return curr_proxy_list

    def create_proxy_object(self, dataset):
        # Check Field[0] for tags and field[1] for values!
        ip = ""
        port = None
        anonymity = AnonymityLevel.UNKNOWN
        country = None
        protocols = []
        for field in dataset:
            if field[0] == 'IP Address':
                # Make sure it is a Valid IP
                ip = field[1].strip()  # String strip()
                # Make sure it is a Valid IP
                if not UrlParser.valid_ip(ip):
                    logger.debug("IP with Invalid format: {}".format(ip))
                    return None
            elif field[0] == 'Port':
                port = field[1].strip()  # String strip()
            elif field[0] == 'Anonymity':
                anonymity = AnonymityLevel.get(field[1].strip())  # String strip()
            elif field[0] == 'Country':
                country = field[1].strip()  # String strip()
            elif field[0] == 'Https':
                if field[1].strip().lower() == 'yes': protocols.extend([Protocol.HTTP, Protocol.HTTPS])
                elif field[1].strip().lower() == 'no': protocols.append(Protocol.HTTP)
        return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country, protocols=protocols)

    def __str__(self):
        return "{0} parser of '{1}' with required bandwidth: '{2}' KBs" \
            .format(self.id, self.url, self.minimum_bandwidth_in_KBs)
import unittest
import sys
import os
from httmock import HTTMock
from tests.mocks import free_proxy_mock, proxy_for_eu_mock, rebro_weebly_mock, prem_mock, sslproxy_mock
from tests.mocks import free_proxy_expected, proxy_for_eu_expected, rebro_weebly_expected, prem_expected, prem_js_mock, sslproxy_expected
from http_request_randomizer.requests.parsers.FreeProxyParser import FreeProxyParser
from http_request_randomizer.requests.parsers.ProxyForEuParser import ProxyForEuParser
from http_request_randomizer.requests.parsers.RebroWeeblyParser import RebroWeeblyParser
from http_request_randomizer.requests.parsers.PremProxyParser import PremProxyParser
from http_request_randomizer.requests.parsers.SslProxyParser import SslProxyParser
from __future__ import absolute_import


sys.path.insert(0, os.path.abspath('.'))


__author__ = 'pgaref'


class TestProxyProviders(unittest.TestCase):

    def test_FreeProxyParser(self):
        with HTTMock(free_proxy_mock):
            proxy_provider = FreeProxyParser('FreeProxy', 'http://free-proxy-list.net')
            proxy_list = proxy_provider.parse_proxyList()
            proxy_list_addr = []
            for proxy in proxy_list:
                proxy_list_addr.append(proxy.get_address())
        self.assertEqual(proxy_list_addr, free_proxy_expected)

    def test_ProxyForEuParser(self):
        with HTTMock(proxy_for_eu_mock):
            proxy_provider = ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0)
            proxy_list = proxy_provider.parse_proxyList()
            proxy_list_addr = []
            for proxy in proxy_list:
                proxy_list_addr.append(proxy.get_address())
        self.assertEqual(proxy_list_addr, proxy_for_eu_expected)

    def test_RebroWeeblyParser(self):

        with HTTMock(rebro_weebly_mock):