#!/usr/bin/env python3

"""
read a directory of html files and try
to convert to jekyll compatible markdown files
"""

import datetime
import glob
import re
import subprocess

from bs4 import BeautifulSoup
import yaml

# These regexes will replace code blocks in HTML within divs with
# fenced code blocks in markdown
#
# This one is for code blocks without language specified
re_highlight_nolang = re.compile(
    r''' {.highlighter-rouge}
::: {.highlight}
``` {.highlight}''',
    re.MULTILINE
)

# And this one is for code blocks with the language specified
re_highlight_lang = re.compile(
    r'''::: {.language-(?P<lang>.*?) \.highlighter-rouge}
::: {.highlight}
``` {.highlight}''',
    re.MULTILINE
)


class Post:
    """one blog post"""

    def __init__(self, slug):
        """load the html content"""
        self.slug = slug
        self._load_html()

    @property
    def markdown_filename(self):
        """construct markdown filename from post date and slug"""
        return "out/{}-{}.md".format(
            self.date,
            self.slug
        )

    @property
    def html_filename(self):
        """construct html filename from slug"""
        return "in/{}.html".format(self.slug.lstrip('/'))

    @property
    def markdown_content(self):
        """html article converted to markdown using pandoc"""
        pandoc_output = subprocess.run(
            [
                '/usr/bin/pandoc',
                '--from=html',
                '--to=markdown',
                '-'
            ],
            input=self._html_content.encode('utf-8'),
            capture_output=True,
            check=True
        )
        markdown_code = pandoc_output.stdout.decode('utf-8')
        with_code_blocks_cleaned = _code_cleanup(markdown_code)
        with_front_matter = "\n".join(
            (self.front_matter, '', with_code_blocks_cleaned)
        )
        return with_front_matter

    @property
    def date(self):
        """the post date"""
        date_string = self._full_html_content.find(
            'time',
            itemprop='datePublished'
        )['datetime']
        post_date = datetime.datetime.fromisoformat(date_string)\
            .strftime('%Y-%m-%d')
        return post_date

    @property
    def title(self):
        """the post title"""
        return self._full_html_content.find('title').text

    @property
    def tags(self):
        """retrieve tags for current post"""
        post_tags = []
        for tag_link in self._full_html_content.find_all('a', rel='tag'):
            post_tags.append(tag_link.text)
        return post_tags

    @property
    def front_matter(self):
        """supply the post's metadata as yaml front matter"""
        post_metadata = {
            'permalink': "/{}".format(self.slug),
            'tags': self.tags,
            'layout': "post",
            'title': self.title
        }
        return "\n".join((
            '---',
            yaml.dump(post_metadata).strip(),
            '---'
        ))

    def save(self):
        """save the markdown file"""
        with open(self.markdown_filename, 'w') as output_file:
            output_file.write(self.markdown_content)
        print("Saved {}".format(self.markdown_filename))

    def _load_html(self):
        """load the post's html code and parse it"""
        with open(self.html_filename, 'r') as input_file:
            full_html_content = input_file.read()
            soup = BeautifulSoup(full_html_content, 'html.parser')
        self._full_html_content = soup
        self._html_content = soup.find('div', itemprop='articleBody')


def _code_cleanup(markdown_code):
    """clean up source code blocks somewhat"""
    # convert divs with code to fenced code blocks
    cleaned_without_lang = re_highlight_nolang.sub('```', markdown_code)
    cleaned_with_lang = re_highlight_lang\
        .sub(r'```\1', cleaned_without_lang)
    # remove unused classes
    without_highlighter = cleaned_with_lang\
        .replace('{.highlighter-rouge}', '')
    # remove divs from code blocks
    cleaned_code_blocks = without_highlighter\
        .replace(':::```', '```')
    # remove wrapping div for article content
    without_wrapper_div = cleaned_code_blocks\
        .replace('::: {.post-content itemprop="articleBody"}', '')
    # remove divs generally
    without_colons = "\n".join(
        [
            line for line in without_wrapper_div.splitlines()
            if line != ":::"
        ]
    ).strip()
    return without_colons


if __name__ == '__main__':
    all_files = glob.glob('in/*.html')
    all_slugs = [filename[3:-5] for filename in all_files]
    try:
        all_slugs.remove('index') # index.html is generated by jekyll
    except ValueError:
        pass
    for post_slug in all_slugs:
        post = Post(post_slug)
        post.save()

#!/usr/bin/env python3

"""
read a directory of html files and try
to convert to jekyll compatible markdown files
"""

import datetime
import glob
import re
import subprocess

from bs4 import BeautifulSoup
import yaml

# These regexes will replace code blocks in HTML within divs with
# fenced code blocks in markdown
#
# This one is for code blocks without language specified
re_highlight_nolang = re.compile(
    r''' {.highlighter-rouge}
::: {.highlight}
``` {.highlight}''',
    re.MULTILINE
)

# And this one is for code blocks with the language specified
re_highlight_lang = re.compile(
    r'''::: {.language-(?P<lang>.*?) \.highlighter-rouge}
::: {.highlight}
``` {.highlight}''',
    re.MULTILINE
)


class Post:
    """one blog post"""

    def __init__(self, slug):
        """load the html content"""
        self.slug = slug
        self._load_html()

    @property
    def markdown_filename(self):
        """construct markdown filename from post date and slug"""
        return "out/{}-{}.md".format(
            self.date,
            self.slug
        )

    @property
    def html_filename(self):
        """construct html filename from slug"""
        return "in/{}.html".format(self.slug.lstrip('/'))

    @property
    def markdown_content(self):
        """html article converted to markdown using pandoc"""
        pandoc_output = subprocess.run(
            [
                '/usr/bin/pandoc',
                '--from=html',
                '--to=markdown',
                '-'
            ],
            input=self._html_content.encode('utf-8'),
            capture_output=True,
            check=True
        )
        markdown_code = pandoc_output.stdout.decode('utf-8')
        with_code_blocks_cleaned = _code_cleanup(markdown_code)
        with_front_matter = "\n".join(
            (self.front_matter, '', with_code_blocks_cleaned)
        )
        return with_front_matter

    @property
    def date(self):
        """the post date"""
        date_string = self._full_html_content.find(
            'time',
            itemprop='datePublished'
        )['datetime']
        post_date = datetime.datetime.fromisoformat(date_string)\
            .strftime('%Y-%m-%d')
        return post_date

    @property
    def title(self):
        """the post title"""
        return self._full_html_content.find('title').text

    @property
    def tags(self):
        """retrieve tags for current post"""
        post_tags = []
        for tag_link in self._full_html_content.find_all('a', rel='tag'):
            post_tags.append(tag_link.text)
        return post_tags

    @property
    def front_matter(self):
        """supply the post's metadata as yaml front matter"""
        post_metadata = {
            'permalink': "/{}".format(self.slug),
            'tags': self.tags,
            'layout': "post",
            'title': self.title
        }
        return "\n".join((
            '---',
            yaml.dump(post_metadata).strip(),
            '---'
        ))

    def save(self):
        """save the markdown file"""
        with open(self.markdown_filename, 'w') as output_file:
            output_file.write(self.markdown_content)
        print("Saved {}".format(self.markdown_filename))

    def _load_html(self):
        """load the post's html code and parse it"""
        with open(self.html_filename, 'r') as input_file:
            full_html_content = input_file.read()
            soup = BeautifulSoup(full_html_content, 'html.parser')
        self._full_html_content = soup
        self._html_content = soup.find('div', itemprop='articleBody')


def _code_cleanup(markdown_code):
    """clean up source code blocks somewhat"""
    # convert divs with code to fenced code blocks
    cleaned_without_lang = re_highlight_nolang.sub('```', markdown_code)
    cleaned_with_lang = re_highlight_lang\
        .sub(r'```\1', cleaned_without_lang)
    # remove unused classes
    without_highlighter = cleaned_with_lang\
        .replace('{.highlighter-rouge}', '')
    # remove divs from code blocks
    cleaned_code_blocks = without_highlighter\
        .replace(':::```', '```')
    # remove wrapping div for article content
    without_wrapper_div = cleaned_code_blocks\
        .replace('::: {.post-content itemprop="articleBody"}', '')
    # remove divs generally
    without_colons = "\n".join(
        [
            line for line in without_wrapper_div.splitlines()
            if line != ":::"
        ]
    ).strip()
    return without_colons


if __name__ == '__main__':
    all_files = glob.glob('in/*.html')
    all_slugs = [filename[3:-5] for filename in all_files]
    try:
        all_slugs.remove('index') # index.html is generated by jekyll
    except ValueError:
        pass
    for post_slug in all_slugs:
        post = Post(post_slug)
        post.save()

#!/usr/bin/env python3 read a directory of html files and try to convert to jekyll compatible markdown files # These regexes will replace code blocks in HTML within divs with # fenced code blocks in markdown # # This one is for code blocks without language specified {.highlighter-rouge} ::: {.highlight} ``` {.highlight} # And this one is for code blocks with the language specified ::: {.language-(?P<lang>.*?) \.highlighter-rouge} ::: {.highlight} ``` {.highlight} one blog post load the html content construct markdown filename from post date and slug construct html filename from slug html article converted to markdown using pandoc the post date the post title retrieve tags for current post supply the post's metadata as yaml front matter save the markdown file load the post's html code and parse it clean up source code blocks somewhat # convert divs with code to fenced code blocks # remove unused classes # remove divs from code blocks # remove wrapping div for article content # remove divs generally # index.html is generated by jekyll