<reponame>microsoft/dstoolkit-ml-ops-for-databricks
"""
This module is used to create the documentation of the library
in a compatible way with Azure DevOps Wiki.

## Prerequisites

Make sure that you have installed pandoc in Ubuntu

```bash
sudo apt-get install pandoc
```
"""

import pdoc
from pdoc import Module
from pypandoc import convert_text
import pypandoc as pp
import re
from typing import List
import sys
from pathlib import Path




def recursive_mods(mod: Module) -> List[Module]:
    """
    Gets all the submodules of the given module

    Parameters
    ----------
    mod : pdoc.Module
        The father module

    Yields
    -------
    [pdoc.Module]
        List of the module with the submodules
    """
    yield mod
    for submod in mod.submodules():
        yield from recursive_mods(submod)


def normalize_md(text: str) -> str:
    """
    Applies a series of normalization steps to a text

    Parameters
    ----------
    text : str
        Text to manipulate

    Returns
    -------
    str
        The new text
    """
    # Replace \r\n with \n
    res = re.sub(r"(\r\n)", '\n', text)
    # Remove multiple lines
    res = re.sub(r"(\n{2,})", '\n', res)
    # Fix title =
    res = re.sub(r"(\n\n)=", '\n', res)
    # Fix title -
    res = re.sub(r"(\n\n)-", '\n', res)
    # Fix parameters
    res = re.sub(r"(^\*{2,})", '\n**', res)
    return res


def normalize_pandoc(text: str) -> str:
    """
    Series of normalization steps for pandoc generated text

    Parameters
    ----------
    text : str
        Pandoc text

    Returns
    -------
    str
        Normalized pandoc text
    """
    text_norm = re.sub(r"(\r\n)", '\n', text)
    text_norm = re.sub(r"( {4})", '\t', text_norm)
    text_norm = re.sub(r'(-----\n\nGenerated by.*)', '----', text_norm)
    return text_norm


def set_methods_ado_links(text: str) -> str:
    """
    Add links to the markdown text compatible with Azure DevOps

    Parameters
    ----------
    text : str
        [description]

    Returns
    -------
    str
        [description]
    """
    text_links = text
    links = set(re.findall(r"\(#(.*)\)", text))
    for link in links:
        text_links = text_links.replace("(#{})".format(link), "(#module-`{}`)".format(link))
    return text_links


def convert_format(text: str, f='markdown+abbreviations', t='commonmark') -> str:
    """
    Converts a text from a format to another

    Parameters
    ----------
    text : str
        The text to convert
    f : str, optional
        The from format, by default 'markdown+abbreviations'
    t : str, optional
        [description], by default 'commonmark'

    Returns
    -------
    str
        [description]
    """
    return convert_text(text, format=f, to=t)


def modules_documentation_pdf(modules: List[Module], f='markdown+abbreviations', t='commonmark') -> str:
    doc_string = pdoc._render_template(template_name='/pdf.mako', modules=modules)
    md_raw = convert_format(doc_string)
    md = normalize_pandoc(md_raw)
    md = set_methods_ado_links(md)
    return md


# def modules_documentation_text(modules: [pdoc.Module], f='markdown+abbreviations', t='commonmark') -> str:
#     """
#     An attempt to create the documentation starting from the text extracted from pdoc.
#     !Not to use

#     Parameters
#     ----------
#     modules : list(pdoc.Module)
#         List of modules from which extract the documentation
#     f : str, optional
#         From pandoc format, by default 'markdown+abbreviations'
#     t : str, optional
#         To pandoc format, by default 'commonmark'

#     Returns
#     -------
#     str
#         [description]
#     """
#     docs = extract_documentation_as_text(modules)
#     md_raw = convert_text(docs, format=f, to=t)
#     md = normalize_pandoc(md_raw)
#     md = set_methods_ado_links(md)
#     return md


def extract_documentation_as_text(modules: List[Module]) -> str:
    """
    Extracts the documentation as text from the given modules.
    Than these are merged with a separator.

    Parameters
    ----------
    modules : list(pdoc.Modules)
        The modules for which create the documentation

    Returns
    -------
    str
        The documentation as plain text
    """
    module_texts = "___".join([x.text() for x in modules])
    return module_texts


def extract_modules(modules_parent: str) -> List[Module]:
    """
    Extracts recursively all the modules that are part of the given module

    Parameters
    ----------
    modules_parent : str
        Name of the module from which start the extraction

    Returns
    -------
    [pdoc.Module]
        List of pdoc modules
    """
    context = pdoc.Context()
    pdoc.link_inheritance(context)
    mod = pdoc.Module(modules_parent, context=context)
    modules_ls = [m for m in recursive_mods(mod)]
    return modules_ls


def test_markdown_convertions(text: str, module_name: str):
    pandoc_from, pandoc_to = pp.get_pandoc_formats()
    res = []
    for f in [x for x in pandoc_from if 'mark' in x]:
        for t in [x for x in pandoc_to if 'mark' in x]:
            text_new = normalize_pandoc(convert_format(text, f, t))
            res.append((text_new, f, t))
            with open('docs_test\\test_doc_{}_{}_{}.md'.format(module_name, f, t), 'w', encoding="utf-8") as out:
                out.write(text_new)

    return True


def create_adow_documentation(package: str) -> str:
    """
    Creates the documentation to use with Azure DevOps Wiki (adow)

    Parameters
    ----------
    module : str
        The name of the module for which create the documentation

    Returns
    -------
    str
        The documentation
    """
    modules = extract_modules(package)
    modules_documentation = [modules_documentation_pdf([module]) for module in modules]
    package_documentation = "".join(modules_documentation)
    return package_documentation


def generate_documentation(package_parent_path: str, package_name: str, output_name: str) -> Path:
    """
    Generates and saves the documentation

    Parameters
    ----------
    package_root_path : str
        Root folder of the package
    package_name : str
        Name of the package
    output_name : str
        Output path of the file
    """
    output_path = Path(package_parent_path).joinpath(output_name)
    sys.path.append(package_parent_path)
    doc = create_adow_documentation(package_name)
    with open(str(output_path), 'w+', encoding="utf-8") as out:
        out.write(doc)
    return output_path

"""
This module is used to create the documentation of the library
in a compatible way with Azure DevOps Wiki.

## Prerequisites

Make sure that you have installed pandoc in Ubuntu

```bash
sudo apt-get install pandoc
```
"""

import pdoc
from pdoc import Module
from pypandoc import convert_text
import pypandoc as pp
import re
from typing import List
import sys
from pathlib import Path




def recursive_mods(mod: Module) -> List[Module]:
    """
    Gets all the submodules of the given module

    Parameters
    ----------
    mod : pdoc.Module
        The father module

    Yields
    -------
    [pdoc.Module]
        List of the module with the submodules
    """
    yield mod
    for submod in mod.submodules():
        yield from recursive_mods(submod)


def normalize_md(text: str) -> str:
    """
    Applies a series of normalization steps to a text

    Parameters
    ----------
    text : str
        Text to manipulate

    Returns
    -------
    str
        The new text
    """
    # Replace \r\n with \n
    res = re.sub(r"(\r\n)", '\n', text)
    # Remove multiple lines
    res = re.sub(r"(\n{2,})", '\n', res)
    # Fix title =
    res = re.sub(r"(\n\n)=", '\n', res)
    # Fix title -
    res = re.sub(r"(\n\n)-", '\n', res)
    # Fix parameters
    res = re.sub(r"(^\*{2,})", '\n**', res)
    return res


def normalize_pandoc(text: str) -> str:
    """
    Series of normalization steps for pandoc generated text

    Parameters
    ----------
    text : str
        Pandoc text

    Returns
    -------
    str
        Normalized pandoc text
    """
    text_norm = re.sub(r"(\r\n)", '\n', text)
    text_norm = re.sub(r"( {4})", '\t', text_norm)
    text_norm = re.sub(r'(-----\n\nGenerated by.*)', '----', text_norm)
    return text_norm


def set_methods_ado_links(text: str) -> str:
    """
    Add links to the markdown text compatible with Azure DevOps

    Parameters
    ----------
    text : str
        [description]

    Returns
    -------
    str
        [description]
    """
    text_links = text
    links = set(re.findall(r"\(#(.*)\)", text))
    for link in links:
        text_links = text_links.replace("(#{})".format(link), "(#module-`{}`)".format(link))
    return text_links


def convert_format(text: str, f='markdown+abbreviations', t='commonmark') -> str:
    """
    Converts a text from a format to another

    Parameters
    ----------
    text : str
        The text to convert
    f : str, optional
        The from format, by default 'markdown+abbreviations'
    t : str, optional
        [description], by default 'commonmark'

    Returns
    -------
    str
        [description]
    """
    return convert_text(text, format=f, to=t)


def modules_documentation_pdf(modules: List[Module], f='markdown+abbreviations', t='commonmark') -> str:
    doc_string = pdoc._render_template(template_name='/pdf.mako', modules=modules)
    md_raw = convert_format(doc_string)
    md = normalize_pandoc(md_raw)
    md = set_methods_ado_links(md)
    return md


# def modules_documentation_text(modules: [pdoc.Module], f='markdown+abbreviations', t='commonmark') -> str:
#     """
#     An attempt to create the documentation starting from the text extracted from pdoc.
#     !Not to use

#     Parameters
#     ----------
#     modules : list(pdoc.Module)
#         List of modules from which extract the documentation
#     f : str, optional
#         From pandoc format, by default 'markdown+abbreviations'
#     t : str, optional
#         To pandoc format, by default 'commonmark'

#     Returns
#     -------
#     str
#         [description]
#     """
#     docs = extract_documentation_as_text(modules)
#     md_raw = convert_text(docs, format=f, to=t)
#     md = normalize_pandoc(md_raw)
#     md = set_methods_ado_links(md)
#     return md


def extract_documentation_as_text(modules: List[Module]) -> str:
    """
    Extracts the documentation as text from the given modules.
    Than these are merged with a separator.

    Parameters
    ----------
    modules : list(pdoc.Modules)
        The modules for which create the documentation

    Returns
    -------
    str
        The documentation as plain text
    """
    module_texts = "___".join([x.text() for x in modules])
    return module_texts


def extract_modules(modules_parent: str) -> List[Module]:
    """
    Extracts recursively all the modules that are part of the given module

    Parameters
    ----------
    modules_parent : str
        Name of the module from which start the extraction

    Returns
    -------
    [pdoc.Module]
        List of pdoc modules
    """
    context = pdoc.Context()
    pdoc.link_inheritance(context)
    mod = pdoc.Module(modules_parent, context=context)
    modules_ls = [m for m in recursive_mods(mod)]
    return modules_ls


def test_markdown_convertions(text: str, module_name: str):
    pandoc_from, pandoc_to = pp.get_pandoc_formats()
    res = []
    for f in [x for x in pandoc_from if 'mark' in x]:
        for t in [x for x in pandoc_to if 'mark' in x]:
            text_new = normalize_pandoc(convert_format(text, f, t))
            res.append((text_new, f, t))
            with open('docs_test\\test_doc_{}_{}_{}.md'.format(module_name, f, t), 'w', encoding="utf-8") as out:
                out.write(text_new)

    return True


def create_adow_documentation(package: str) -> str:
    """
    Creates the documentation to use with Azure DevOps Wiki (adow)

    Parameters
    ----------
    module : str
        The name of the module for which create the documentation

    Returns
    -------
    str
        The documentation
    """
    modules = extract_modules(package)
    modules_documentation = [modules_documentation_pdf([module]) for module in modules]
    package_documentation = "".join(modules_documentation)
    return package_documentation


def generate_documentation(package_parent_path: str, package_name: str, output_name: str) -> Path:
    """
    Generates and saves the documentation

    Parameters
    ----------
    package_root_path : str
        Root folder of the package
    package_name : str
        Name of the package
    output_name : str
        Output path of the file
    """
    output_path = Path(package_parent_path).joinpath(output_name)
    sys.path.append(package_parent_path)
    doc = create_adow_documentation(package_name)
    with open(str(output_path), 'w+', encoding="utf-8") as out:
        out.write(doc)
    return output_path
This module is used to create the documentation of the library in a compatible way with Azure DevOps Wiki.  ## Prerequisites  Make sure that you have installed pandoc in Ubuntu  ```bash sudo apt-get install pandoc ``` Gets all the submodules of the given module      Parameters     ----------     mod : pdoc.Module         The father module      Yields     -------     [pdoc.Module]         List of the module with the submodules Applies a series of normalization steps to a text      Parameters     ----------     text : str         Text to manipulate      Returns     -------     str         The new text # Replace \r\n with \n # Remove multiple lines # Fix title = # Fix title - # Fix parameters Series of normalization steps for pandoc generated text      Parameters     ----------     text : str         Pandoc text      Returns     -------     str         Normalized pandoc text Add links to the markdown text compatible with Azure DevOps      Parameters     ----------     text : str         [description]      Returns     -------     str         [description] #(.*)\)", text)) #{})".format(link), "(#module-`{}`)".format(link)) Converts a text from a format to another      Parameters     ----------     text : str         The text to convert     f : str, optional         The from format, by default 'markdown+abbreviations'     t : str, optional         [description], by default 'commonmark'      Returns     -------     str         [description] # def modules_documentation_text(modules: [pdoc.Module], f='markdown+abbreviations', t='commonmark') -> str: #     """ #     An attempt to create the documentation starting from the text extracted from pdoc. #     !Not to use #     Parameters #     ---------- #     modules : list(pdoc.Module) #         List of modules from which extract the documentation #     f : str, optional #         From pandoc format, by default 'markdown+abbreviations' #     t : str, optional #         To pandoc format, by default 'commonmark' #     Returns #     ------- #     str #         [description] #     """ #     docs = extract_documentation_as_text(modules) #     md_raw = convert_text(docs, format=f, to=t) #     md = normalize_pandoc(md_raw) #     md = set_methods_ado_links(md) #     return md Extracts the documentation as text from the given modules.     Than these are merged with a separator.      Parameters     ----------     modules : list(pdoc.Modules)         The modules for which create the documentation      Returns     -------     str         The documentation as plain text Extracts recursively all the modules that are part of the given module      Parameters     ----------     modules_parent : str         Name of the module from which start the extraction      Returns     -------     [pdoc.Module]         List of pdoc modules Creates the documentation to use with Azure DevOps Wiki (adow)      Parameters     ----------     module : str         The name of the module for which create the documentation      Returns     -------     str         The documentation Generates and saves the documentation      Parameters     ----------     package_root_path : str         Root folder of the package     package_name : str         Name of the package     output_name : str         Output path of the file