# meds/utils/url.py
#
#

""" functions that fetch data from url """

from meds.utils.trace import get_exception
from meds import __version__

import sys, os

#sys.path.insert(0, os.getcwd())

try: import feedparser ; gotparser = True
except: gotparser = False

import logging
import urllib
import http
import html
import sys
import re

def get_feed(url):
    from meds.object import Object
    if not gotparser:
        logging.error("no feedparser available")
        return []
    if not "http" in url:
        logging.error("%s is not an url." % url)
        return []
    try: result = feedparser.parse(get_url(url))
    except Exception as ex:
        logging.error(get_exception(url))
        return
    if "entries" in result:
        for entry in result["entries"][::-1]:
            yield Object(entry)

def get_url(*args, **kwargs):
    url = urllib.parse.urlunparse(urllib.parse.urlparse(args[0]))
    req = urllib.request.Request(url, headers={"User-Agent": useragent()})
    resp = urllib.request.urlopen(req)
    data = resp.read()
    logging.info("! %s %s %s" % (resp.status, resp.reason, url))
    return data

def get_url2(url, myheaders={}, postdata={}, keyfile=None, certfile="", port=80):
    headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8', 'Accept': 'text/plain; text/html; application/json', 'User-Agent': useragent()}
    headers.update(myheaders)
    urlparts = urllib.parse.urlparse(url)
    if "https" in url: connection = http.client.HTTPSConnection(urlparts[1]) # keyfile, certfile)
    else: connection = http.client.HTTPConnection(urlparts[1])
    connection.connect()
    connection.request("GET", urlparts[2], None, headers)
    resp = connection.getresponse()
    data = resp.read()
    logging.warn("# %s %s %s" % (resp.status, resp.reason, url))
    connection.close()
    return data

def need_redirect(resp):
    if resp.status == 301 or resp.status == 302: url = resp.getheader("Location") ; return url

def useragent(): return 'Mozilla/5.0 (X11; Linux x86_64) MEDS %s +http://pikacode.com/bart/meds)' % __version__

def unescape(text): return html.parser.HTMLParser().unescape(text)

def extract_div(search, data):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(data) 
    divs = soup('div')
    for div in divs:
       if div.get(search): return div

def get_encoding(data):
    if hasattr(data, 'info') and 'content-type' in data.info and 'charset' in data.info['content-type'].lower():
        charset = data.info['content-type'].lower().split('charset', 1)[1].strip()
        if charset[0] == '=':
            charset = charset[1:].strip()
            if ';' in charset: return charset.split(';')[0].strip()
            return charset
    if '<meta' in data.lower():
        metas = re.findall('<meta[^>]+>', data, re.I | re.M)
        if metas:
            for meta in metas:
                test_http_equiv = re.search('http-equiv\s*=\s*[\'"]([^\'"]+)[\'"]', meta, re.I)
                if test_http_equiv and test_http_equiv.group(1).lower() == 'content-type':
                    test_content = re.search('content\s*=\s*[\'"]([^\'"]+)[\'"]', meta, re.I)
                    if test_content:
                        test_charset = re.search('charset\s*=\s*([^\s\'"]+)', meta, re.I)
                        if test_charset: return test_charset.group(1)
    try:
        test = chardet.detect(data)
        if 'encoding' in test: return test['encoding']
    except: pass
    return sys.getdefaultencoding()

def parse_url(*args, **kwargs):
    """

    Attribute       Index   Value                   Value if not present
    scheme          0       URL scheme specifier    empty string
    netloc          1       Network location part   empty string
    path            2       Hierarchical path       empty string
    query           3       Query component         empty string
    fragment        4       Fragment identifier     empty string

    """
    url = args[0]
    parsed = urllib.parse.urlsplit(url)
    target = parsed[2].split("/")
    if "." in target[-1]: basepath = "/".join(target[:-1]) ; file = target[-1]
    else: basepath = parsed[2] ; file = None
    if basepath.endswith("/"): basepath = basepath[:-1]
    base = urllib.parse.urlunsplit((parsed[0], parsed[1], basepath , "", ""))
    root = urllib.parse.urlunsplit((parsed[0], parsed[1], "", "", ""))
    return (basepath, base, root, file)

def strip_html(text):
    import bs4
    if text.startswith("http"): return text
    try: soup = bs4.BeautifulSoup(text, "lxml")
    except: soup = bs4.BeautifulSoup(text)
    res = ""
    for chunk in soup.findAll(text=True):
        if isinstance(chunk, bs4.CData): res += str(chunk.content[0]) + " "
        else: res += str(chunk) + " "
    return res

# meds/utils/url.py
#
#

""" functions that fetch data from url """

from meds.utils.trace import get_exception
from meds import __version__

import sys, os

#sys.path.insert(0, os.getcwd())

try: import feedparser ; gotparser = True
except: gotparser = False

import logging
import urllib
import http
import html
import sys
import re

def get_feed(url):
    from meds.object import Object
    if not gotparser:
        logging.error("no feedparser available")
        return []
    if not "http" in url:
        logging.error("%s is not an url." % url)
        return []
    try: result = feedparser.parse(get_url(url))
    except Exception as ex:
        logging.error(get_exception(url))
        return
    if "entries" in result:
        for entry in result["entries"][::-1]:
            yield Object(entry)

def get_url(*args, **kwargs):
    url = urllib.parse.urlunparse(urllib.parse.urlparse(args[0]))
    req = urllib.request.Request(url, headers={"User-Agent": useragent()})
    resp = urllib.request.urlopen(req)
    data = resp.read()
    logging.info("! %s %s %s" % (resp.status, resp.reason, url))
    return data

def get_url2(url, myheaders={}, postdata={}, keyfile=None, certfile="", port=80):
    headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8', 'Accept': 'text/plain; text/html; application/json', 'User-Agent': useragent()}
    headers.update(myheaders)
    urlparts = urllib.parse.urlparse(url)
    if "https" in url: connection = http.client.HTTPSConnection(urlparts[1]) # keyfile, certfile)
    else: connection = http.client.HTTPConnection(urlparts[1])
    connection.connect()
    connection.request("GET", urlparts[2], None, headers)
    resp = connection.getresponse()
    data = resp.read()
    logging.warn("# %s %s %s" % (resp.status, resp.reason, url))
    connection.close()
    return data

def need_redirect(resp):
    if resp.status == 301 or resp.status == 302: url = resp.getheader("Location") ; return url

def useragent(): return 'Mozilla/5.0 (X11; Linux x86_64) MEDS %s +http://pikacode.com/bart/meds)' % __version__

def unescape(text): return html.parser.HTMLParser().unescape(text)

def extract_div(search, data):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(data) 
    divs = soup('div')
    for div in divs:
       if div.get(search): return div

def get_encoding(data):
    if hasattr(data, 'info') and 'content-type' in data.info and 'charset' in data.info['content-type'].lower():
        charset = data.info['content-type'].lower().split('charset', 1)[1].strip()
        if charset[0] == '=':
            charset = charset[1:].strip()
            if ';' in charset: return charset.split(';')[0].strip()
            return charset
    if '<meta' in data.lower():
        metas = re.findall('<meta[^>]+>', data, re.I | re.M)
        if metas:
            for meta in metas:
                test_http_equiv = re.search('http-equiv\s*=\s*[\'"]([^\'"]+)[\'"]', meta, re.I)
                if test_http_equiv and test_http_equiv.group(1).lower() == 'content-type':
                    test_content = re.search('content\s*=\s*[\'"]([^\'"]+)[\'"]', meta, re.I)
                    if test_content:
                        test_charset = re.search('charset\s*=\s*([^\s\'"]+)', meta, re.I)
                        if test_charset: return test_charset.group(1)
    try:
        test = chardet.detect(data)
        if 'encoding' in test: return test['encoding']
    except: pass
    return sys.getdefaultencoding()

def parse_url(*args, **kwargs):
    """

    Attribute       Index   Value                   Value if not present
    scheme          0       URL scheme specifier    empty string
    netloc          1       Network location part   empty string
    path            2       Hierarchical path       empty string
    query           3       Query component         empty string
    fragment        4       Fragment identifier     empty string

    """
    url = args[0]
    parsed = urllib.parse.urlsplit(url)
    target = parsed[2].split("/")
    if "." in target[-1]: basepath = "/".join(target[:-1]) ; file = target[-1]
    else: basepath = parsed[2] ; file = None
    if basepath.endswith("/"): basepath = basepath[:-1]
    base = urllib.parse.urlunsplit((parsed[0], parsed[1], basepath , "", ""))
    root = urllib.parse.urlunsplit((parsed[0], parsed[1], "", "", ""))
    return (basepath, base, root, file)

def strip_html(text):
    import bs4
    if text.startswith("http"): return text
    try: soup = bs4.BeautifulSoup(text, "lxml")
    except: soup = bs4.BeautifulSoup(text)
    res = ""
    for chunk in soup.findAll(text=True):
        if isinstance(chunk, bs4.CData): res += str(chunk.content[0]) + " "
        else: res += str(chunk) + " "
    return res

# meds/utils/url.py # # functions that fetch data from url #sys.path.insert(0, os.getcwd()) # keyfile, certfile) Attribute       Index   Value                   Value if not present     scheme          0       URL scheme specifier    empty string     netloc          1       Network location part   empty string     path            2       Hierarchical path       empty string     query           3       Query component         empty string     fragment        4       Fragment identifier     empty string