import requests

"""
A webscraping tool that looks for information from a URL based
on the keyword argument 'get_info_of'.
Currently only tested on
https://iris.lmsal.com/iris2/iris2_chapter02_01.html#inversion-of-iris-mg-ii-h-k-lines-with-iris2
because of its relevance.
Definitely not the most elegant approach, but works fine for now.
"""


def request_info(URL, get_info_of):
    """
    URL is URL.
    get_info_of is a list of parameters to extract info from
    """

    # convert page to utf-8 text format
    page = requests.get(URL)
    page.encoding = page.apparent_encoding
    text = page.text
    out_info = []

    predef = '<li><p><code class="docutils literal notranslate"><span class="pre">'
    for param in get_info_of:
        # locate the keyword from the URL page
        if param=="model": idx_i = text.find(predef+param+"<")
        elif param=="mask_extra2":
            param = "mask_extra"
            idx_i = text.find(predef+param)
            idx_i += text[idx_i+1:].find(predef+param)+1
        else: idx_i = text.find(predef+param)
        #param_text = text[idx_i+len(predef):idx_i+len(predef)+1500]
        param_text = text[idx_i+len(predef):idx_i+len(predef)+1500]
        if param=="qc_flag": idx_f = param_text.find("</p></li>\n</ul>")
        else: idx_f = param_text.find("</p></li>")
        param_text = param_text[:idx_f]

        # lots of removals
        param_text = param_text.replace('%s</span></code>: '%param, "")
        param_text = param_text.replace('<code class="docutils literal notranslate"><span class="pre">', "")
        param_text = param_text.replace('</span></code>', "")
        param_text = param_text.replace('\n', " ")
        param_text = param_text.replace('<span class="math notranslate nohighlight">\(', "")
        param_text = param_text.replace('\)</span>', "")
        param_text = param_text.replace('<a class="reference external" href=', "")
        param_text = param_text.replace('<code class="xref std std-numref docutils literal notranslate"><span class="pre">', "")
        param_text = param_text.replace('</span> <span', "")
        param_text = param_text.replace('class="pre">', "")
        param_text = param_text.replace('<a class="reference internal" href="#inversion-of-a-large-raster-file-a-region-of-interest-or-a-masked-area"><span class="std std-numref">S', "")
        param_text = param_text.replace('</span></a>', "")
        param_text = param_text.replace("</p> <ul> <li><p>", " ")
        param_text = param_text.replace("</p></li> <li><p>", ", ")
        param_text = param_text.replace("\\", "")
        param_text = param_text.replace("\\", "")

        out_info.append(param_text)

    return out_info






if __name__ == '__main__':
    """
    URL = "https://iris.lmsal.com/iris2/iris2_chapter02_01.html#inversion-of-iris-mg-ii-h-k-lines-with-iris2"
    get_info = ["qc_flag"]
    info = request_info(URL, get_info)
    """


    #print(info)

import requests

"""
A webscraping tool that looks for information from a URL based
on the keyword argument 'get_info_of'.
Currently only tested on
https://iris.lmsal.com/iris2/iris2_chapter02_01.html#inversion-of-iris-mg-ii-h-k-lines-with-iris2
because of its relevance.
Definitely not the most elegant approach, but works fine for now.
"""


def request_info(URL, get_info_of):
    """
    URL is URL.
    get_info_of is a list of parameters to extract info from
    """

    # convert page to utf-8 text format
    page = requests.get(URL)
    page.encoding = page.apparent_encoding
    text = page.text
    out_info = []

    predef = '<li><p><code class="docutils literal notranslate"><span class="pre">'
    for param in get_info_of:
        # locate the keyword from the URL page
        if param=="model": idx_i = text.find(predef+param+"<")
        elif param=="mask_extra2":
            param = "mask_extra"
            idx_i = text.find(predef+param)
            idx_i += text[idx_i+1:].find(predef+param)+1
        else: idx_i = text.find(predef+param)
        #param_text = text[idx_i+len(predef):idx_i+len(predef)+1500]
        param_text = text[idx_i+len(predef):idx_i+len(predef)+1500]
        if param=="qc_flag": idx_f = param_text.find("</p></li>\n</ul>")
        else: idx_f = param_text.find("</p></li>")
        param_text = param_text[:idx_f]

        # lots of removals
        param_text = param_text.replace('%s</span></code>: '%param, "")
        param_text = param_text.replace('<code class="docutils literal notranslate"><span class="pre">', "")
        param_text = param_text.replace('</span></code>', "")
        param_text = param_text.replace('\n', " ")
        param_text = param_text.replace('<span class="math notranslate nohighlight">\(', "")
        param_text = param_text.replace('\)</span>', "")
        param_text = param_text.replace('<a class="reference external" href=', "")
        param_text = param_text.replace('<code class="xref std std-numref docutils literal notranslate"><span class="pre">', "")
        param_text = param_text.replace('</span> <span', "")
        param_text = param_text.replace('class="pre">', "")
        param_text = param_text.replace('<a class="reference internal" href="#inversion-of-a-large-raster-file-a-region-of-interest-or-a-masked-area"><span class="std std-numref">S', "")
        param_text = param_text.replace('</span></a>', "")
        param_text = param_text.replace("</p> <ul> <li><p>", " ")
        param_text = param_text.replace("</p></li> <li><p>", ", ")
        param_text = param_text.replace("\\", "")
        param_text = param_text.replace("\\", "")

        out_info.append(param_text)

    return out_info






if __name__ == '__main__':
    """
    URL = "https://iris.lmsal.com/iris2/iris2_chapter02_01.html#inversion-of-iris-mg-ii-h-k-lines-with-iris2"
    get_info = ["qc_flag"]
    info = request_info(URL, get_info)
    """


    #print(info)

A webscraping tool that looks for information from a URL based on the keyword argument 'get_info_of'. Currently only tested on https://iris.lmsal.com/iris2/iris2_chapter02_01.html#inversion-of-iris-mg-ii-h-k-lines-with-iris2 because of its relevance. Definitely not the most elegant approach, but works fine for now. URL is URL.     get_info_of is a list of parameters to extract info from # convert page to utf-8 text format # locate the keyword from the URL page #param_text = text[idx_i+len(predef):idx_i+len(predef)+1500] # lots of removals URL = "https://iris.lmsal.com/iris2/iris2_chapter02_01.html#inversion-of-iris-mg-ii-h-k-lines-with-iris2"     get_info = ["qc_flag"]     info = request_info(URL, get_info) #print(info)