<reponame>urkesh-org/DABI
"""
Convert .htm files to .md, for DABI consumption
Also generate templates from frameset...
"""

from datetime import datetime
from pathlib import Path
import shutil
import codecs
import re
from bs4 import BeautifulSoup
import logging

from DABI import get_settings, SETTINGS_FILE
from DABI_pelican import parse_authorship
from log import init_logging

logger = logging.getLogger(__name__)
OUTPUT_DIR = Path('_converted_DABI/')

BODY_REGEX = re.compile(r'<body.*?>(?P<body>.+?)(</body>|</html>|$)', flags=re.DOTALL | re.IGNORECASE)


def main():
    init_logging()
    logger.info(f'Starting conversion of .md files and templates at {str(datetime.now().astimezone().isoformat(timespec="seconds"))}.')

    shutil.rmtree(OUTPUT_DIR)  # ! dangerous
    OUTPUT_DIR.mkdir()
    template_path = OUTPUT_DIR / "_templates" / "Mes-pol"
    template_path.mkdir(parents=True)

    #settings = get_settings(SETTINGS_FILE)  # lookup the site directories
    authorship = parse_authorship(Path('../_Databases/Authorship.txt'))
    
    for file in Path('.').glob('**/*.htm'):
        if any(folder.startswith('_') or folder == 'inactive' for folder in file.parts):
            continue
        if not file.is_file():
            continue
        if ' ' in file.name and re.search(r' [A-z]{1,2}[0-9xyz][0-9]{2}', file.name):  # ignore files with date in filename (old versions)
            continue
        
        if file.stem.lower().startswith('sub'):  # frameset file, ignore
            continue
        
        try:
            with file.open(mode='r', encoding='utf-8-sig') as fp:  # parse HTML file
                content = fp.read()
        except UnicodeError:  # try to read from Windows-1252
            with codecs.open(str(file), mode='r', encoding='cp1252') as fp:
                content = fp.read()
        soup = BeautifulSoup(content, 'html.parser')
        
        if file.stem.lower().endswith('toc'):  # ToC file, put in templates
            output_file = template_path / file.with_suffix('.html')
            
            body = soup.find('body').findChildren(recursive=False)[0]
            # TODO: check for sub_nav

            output_text = f'''\
{{% extends "Mes-pol/ToC.html" %}}


{{% block sub_nav %}}

{body}

{{% endblock -%}}
'''

        else:  # file with content, convert to MD
            output_file = OUTPUT_DIR / file.with_suffix('.md')

            try:
                body = BODY_REGEX.search(content).group("body").strip()
            except AttributeError:
                logger.warning(f"{file}: no <body> found")
                body = ''
            
            body = re.sub(r'(<br\\?/?>\s*)+$', '', body, flags=re.IGNORECASE).strip()  # remove endings <br>
            body = re.sub(r'<A NAME=\"?0\"?>(?:</a>)?', '', body, flags=re.IGNORECASE)  # remove <a name=0>
            body = re.sub(r'<a\s+name=\"?(.*?)\"?>(?:</a>)?(.*(?:\n.*){0,3})<h3>(.*?)</h3>', r'\g<2>\n### \g<3> {#\g<1>}', body, flags=re.IGNORECASE)  # convert <h3>
            body = re.sub(r'&nbsp;&nbsp;&nbsp;&nbsp;(?:&nbsp;)', '—', body)  # remove &nbsp;
            #body = re.sub(r'<br\\?/?>', '\n\n', body, flags=re.IGNORECASE)
            
            try:
                title = soup.find('h1').text
            except AttributeError:
                title = ''
            
            try:
                subtitle = soup.find('h2').text
            except AttributeError:
                subtitle = ''
            
            try:
                author = soup.find('h3').text
                for key, value in authorship.items():
                    if value.text == author:
                        author = key
                        break
            except AttributeError:
                author = ''
            
            try:
                date = soup.find_all('h4')[1].text
            except AttributeError and IndexError:
                date = ''
            
            output_text = f'''\
HTML Mes-pol/ToC
T {title}{chr(10)+'S '+subtitle if subtitle else ''} 
AU {author}
D {date}


[TOC]


{body}

'''
            
        # write file
        output_file.parent.mkdir(parents=True, exist_ok=True)
        with output_file.open(mode='w', encoding='utf-8') as outfile:
            outfile.write(output_text)


if __name__ == '__main__':
    main()

"""
Convert .htm files to .md, for DABI consumption
Also generate templates from frameset...
"""

from datetime import datetime
from pathlib import Path
import shutil
import codecs
import re
from bs4 import BeautifulSoup
import logging

from DABI import get_settings, SETTINGS_FILE
from DABI_pelican import parse_authorship
from log import init_logging

logger = logging.getLogger(__name__)
OUTPUT_DIR = Path('_converted_DABI/')

BODY_REGEX = re.compile(r'<body.*?>(?P<body>.+?)(</body>|</html>|$)', flags=re.DOTALL | re.IGNORECASE)


def main():
    init_logging()
    logger.info(f'Starting conversion of .md files and templates at {str(datetime.now().astimezone().isoformat(timespec="seconds"))}.')

    shutil.rmtree(OUTPUT_DIR)  # ! dangerous
    OUTPUT_DIR.mkdir()
    template_path = OUTPUT_DIR / "_templates" / "Mes-pol"
    template_path.mkdir(parents=True)

    #settings = get_settings(SETTINGS_FILE)  # lookup the site directories
    authorship = parse_authorship(Path('../_Databases/Authorship.txt'))
    
    for file in Path('.').glob('**/*.htm'):
        if any(folder.startswith('_') or folder == 'inactive' for folder in file.parts):
            continue
        if not file.is_file():
            continue
        if ' ' in file.name and re.search(r' [A-z]{1,2}[0-9xyz][0-9]{2}', file.name):  # ignore files with date in filename (old versions)
            continue
        
        if file.stem.lower().startswith('sub'):  # frameset file, ignore
            continue
        
        try:
            with file.open(mode='r', encoding='utf-8-sig') as fp:  # parse HTML file
                content = fp.read()
        except UnicodeError:  # try to read from Windows-1252
            with codecs.open(str(file), mode='r', encoding='cp1252') as fp:
                content = fp.read()
        soup = BeautifulSoup(content, 'html.parser')
        
        if file.stem.lower().endswith('toc'):  # ToC file, put in templates
            output_file = template_path / file.with_suffix('.html')
            
            body = soup.find('body').findChildren(recursive=False)[0]
            # TODO: check for sub_nav

            output_text = f'''\
{{% extends "Mes-pol/ToC.html" %}}


{{% block sub_nav %}}

{body}

{{% endblock -%}}
'''

        else:  # file with content, convert to MD
            output_file = OUTPUT_DIR / file.with_suffix('.md')

            try:
                body = BODY_REGEX.search(content).group("body").strip()
            except AttributeError:
                logger.warning(f"{file}: no <body> found")
                body = ''
            
            body = re.sub(r'(<br\\?/?>\s*)+$', '', body, flags=re.IGNORECASE).strip()  # remove endings <br>
            body = re.sub(r'<A NAME=\"?0\"?>(?:</a>)?', '', body, flags=re.IGNORECASE)  # remove <a name=0>
            body = re.sub(r'<a\s+name=\"?(.*?)\"?>(?:</a>)?(.*(?:\n.*){0,3})<h3>(.*?)</h3>', r'\g<2>\n### \g<3> {#\g<1>}', body, flags=re.IGNORECASE)  # convert <h3>
            body = re.sub(r'&nbsp;&nbsp;&nbsp;&nbsp;(?:&nbsp;)', '—', body)  # remove &nbsp;
            #body = re.sub(r'<br\\?/?>', '\n\n', body, flags=re.IGNORECASE)
            
            try:
                title = soup.find('h1').text
            except AttributeError:
                title = ''
            
            try:
                subtitle = soup.find('h2').text
            except AttributeError:
                subtitle = ''
            
            try:
                author = soup.find('h3').text
                for key, value in authorship.items():
                    if value.text == author:
                        author = key
                        break
            except AttributeError:
                author = ''
            
            try:
                date = soup.find_all('h4')[1].text
            except AttributeError and IndexError:
                date = ''
            
            output_text = f'''\
HTML Mes-pol/ToC
T {title}{chr(10)+'S '+subtitle if subtitle else ''} 
AU {author}
D {date}


[TOC]


{body}

'''
            
        # write file
        output_file.parent.mkdir(parents=True, exist_ok=True)
        with output_file.open(mode='w', encoding='utf-8') as outfile:
            outfile.write(output_text)


if __name__ == '__main__':
    main()
Convert .htm files to .md, for DABI consumption Also generate templates from frameset... # ! dangerous #settings = get_settings(SETTINGS_FILE)  # lookup the site directories # ignore files with date in filename (old versions) # frameset file, ignore # parse HTML file # try to read from Windows-1252 # ToC file, put in templates # TODO: check for sub_nav \ {{% extends "Mes-pol/ToC.html" %}}   {{% block sub_nav %}}  {body}  {{% endblock -%}} # file with content, convert to MD # remove endings <br> # remove <a name=0> ### \g<3> {#\g<1>}', body, flags=re.IGNORECASE)  # convert <h3> # remove &nbsp; #body = re.sub(r'<br\\?/?>', '\n\n', body, flags=re.IGNORECASE) \ HTML Mes-pol/ToC T {title}{chr(10)+'S '+subtitle if subtitle else ''}  AU {author} D {date}   [TOC]   {body} # write file