"""Extract and merge links from HTML bookmarks files exported from Firefox.

A Firefox bookmarks file doesn't make bookmarks available in a form that's easy to work with.
This tool solves that problem.
"""

import json
import sys

from bs4 import BeautifulSoup
import click


def firefox_link(address, name, attrs):
    return {
        'address': address,
        'name': name,
        'attrs': attrs
    }


def bookmark_folder(folder_name, bookmarks):
    return {
        'folder_name': folder_name,
        'bookmarks': bookmarks
    }


def bookmarks_to_html(bookmarks):
    header = """<!DOCTYPE NETSCAPE-Bookmark-file-1>
<!-- This is an automatically generated file.
     It will be read and overwritten.
     DO NOT EDIT! -->
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
<TITLE>Bookmarks</TITLE>
<H1>Bookmarks Menu</H1>

<dl><p>"""
    contents = [bookmark_to_html(bookmark) for bookmark in bookmarks]
    footer = "</dl>"
    return '\n'.join([header] + contents + [footer])


def bookmark_to_html(bookmark):
    if 'address' in bookmark:
        return firefox_link_to_html(bookmark)
    elif 'bookmarks' in bookmark:
        return bookmark_folder_to_html(bookmark)
    raise ValueError('Unhandled bookmark type')


def firefox_link_to_html(link):
    tags = " ".join(
        [f'href="{link["address"]}"'] +
        [f'{attr}="{value}"' for attr, value in link['attrs'].items()]
    )
    return f'<dt><a {tags}>{link["name"]}</a>'


def bookmark_folder_to_html(folder):
    # TODO the header also has ADD_DATE, LAST_MODIFIED, but I don't care much about those things.
    header = f'<dt><h3>{folder["folder_name"]}</h3>'
    start = '<dl><p>'
    contents = [bookmark_to_html(bookmark) for bookmark in folder['bookmarks']]
    end = '</dl><p>'
    return '\n'.join([header, start] + contents + [end])


def parse_bookmarks(root):
    """Parse the tree of bookmarks from an HTML bookmark file exported from Firefox.

    The doctype of this file claims to be "NETSCAPE-Bookmark-file-1".
    """
    bookmarks = []
    for e in root.find_all('dt', recursive=False):
        for link in e.find_all('a', recursive=False):
            bookmarks.append(firefox_link(
                link.get('href'),
                link.string,
                {attr: link.get(attr) for attr in link.attrs if attr != 'href'},
            ))

        for folder in e.find_all('h3', recursive=False):
            folder_name = folder.string
            folder_dl = folder.find_next_sibling('dl')
            bookmarks.append(bookmark_folder(
                folder_name,
                parse_bookmarks(folder_dl)
            ))
    return bookmarks


def parse_bookmarks_from_html(filename):
    with open(filename) as f:
        soup = BeautifulSoup(f, 'html5lib')
    return parse_bookmarks(soup.find('dl'))


def bookmark_addresses(bookmarks):
    """Generate addresses from a parsed tree of bookmarks."""
    # A bookmark is either a firefox_link or a bookmark_folder
    for bookmark in bookmarks:
        if 'address' in bookmark:
            yield bookmark['address']
        elif 'bookmarks' in bookmark:
            yield from bookmark_addresses(bookmark['bookmarks'])


def remove_duplicate_bookmarks(bookmarks, addresses):
    """Remove duplicate addresses from a tree of bookmarks."""
    out_bookmarks = []
    for bookmark in bookmarks:
        if 'address' in bookmark:
            if bookmark['address'] in addresses:
                print(f'Dupe: {bookmark["address"]}', file=sys.stderr)
            elif bookmark['address'].startswith('place:'):
                # Skip recents and other autogenerated bookmark lists
                continue
            else:
                out_bookmarks.append(bookmark)
        elif 'bookmarks' in bookmark:
            deduped = remove_duplicate_bookmarks(bookmark['bookmarks'], addresses)
            if deduped:
                out_bookmarks.append(bookmark_folder(bookmark['folder_name'], deduped))
    return out_bookmarks


@click.group(help=__doc__)
def cli():
    pass


@cli.command('extract')
@click.argument('bookmarks_filename')
def click_extract(bookmarks_filename):
    """Extract bookmarks in a bookmarks HTML file to structured JSON."""
    print(json.dumps(parse_bookmarks_from_html(bookmarks_filename), indent=4))


@cli.command('prepare-import')
@click.argument('primary_bookmarks_filename')
@click.argument('secondary_bookmarks_filename')
def click_merge(primary_bookmarks_filename, secondary_bookmarks_filename):
    """Prepare secondary bookmarks for import.

    Given an HTML bookmarks export from the primary Firefox profile and an HTML bookmarks export
    from the secondary Firefox profile, write an HTML file containing bookmarks that appear only in
    the secondary profile.

    The bookmark folder structure from the secondary file is preserved in a new folder with the
    following deduplication operations performed:

    - If a bookmark's address also appears in the primary file, it is omitted from the secondary.
    - If omissions of bookmarks result in empty folders, the empty folders are pruned.

    A new bookmarks file is written out, which can then be imported into the primary Firefox
    profile via the Bookmarks Manager.  The user can then use the Bookmarks Manager to manipulate
    the secondary's folder structure, merging it with the primary with the assurance that doing so
    won't result in adding duplicate bookmarks.
    """
    primary_bookmarks = parse_bookmarks_from_html(primary_bookmarks_filename)
    primary_addresses = set(bookmark_addresses(primary_bookmarks))

    secondary_bookmarks = parse_bookmarks_from_html(secondary_bookmarks_filename)

    deduplicated_secondary_bookmarks = remove_duplicate_bookmarks(
        secondary_bookmarks, primary_addresses)

    print(bookmarks_to_html(deduplicated_secondary_bookmarks))


if __name__ == '__main__':
    cli()

"""Extract and merge links from HTML bookmarks files exported from Firefox.

A Firefox bookmarks file doesn't make bookmarks available in a form that's easy to work with.
This tool solves that problem.
"""

import json
import sys

from bs4 import BeautifulSoup
import click


def firefox_link(address, name, attrs):
    return {
        'address': address,
        'name': name,
        'attrs': attrs
    }


def bookmark_folder(folder_name, bookmarks):
    return {
        'folder_name': folder_name,
        'bookmarks': bookmarks
    }


def bookmarks_to_html(bookmarks):
    header = """<!DOCTYPE NETSCAPE-Bookmark-file-1>
<!-- This is an automatically generated file.
     It will be read and overwritten.
     DO NOT EDIT! -->
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
<TITLE>Bookmarks</TITLE>
<H1>Bookmarks Menu</H1>

<dl><p>"""
    contents = [bookmark_to_html(bookmark) for bookmark in bookmarks]
    footer = "</dl>"
    return '\n'.join([header] + contents + [footer])


def bookmark_to_html(bookmark):
    if 'address' in bookmark:
        return firefox_link_to_html(bookmark)
    elif 'bookmarks' in bookmark:
        return bookmark_folder_to_html(bookmark)
    raise ValueError('Unhandled bookmark type')


def firefox_link_to_html(link):
    tags = " ".join(
        [f'href="{link["address"]}"'] +
        [f'{attr}="{value}"' for attr, value in link['attrs'].items()]
    )
    return f'<dt><a {tags}>{link["name"]}</a>'


def bookmark_folder_to_html(folder):
    # TODO the header also has ADD_DATE, LAST_MODIFIED, but I don't care much about those things.
    header = f'<dt><h3>{folder["folder_name"]}</h3>'
    start = '<dl><p>'
    contents = [bookmark_to_html(bookmark) for bookmark in folder['bookmarks']]
    end = '</dl><p>'
    return '\n'.join([header, start] + contents + [end])


def parse_bookmarks(root):
    """Parse the tree of bookmarks from an HTML bookmark file exported from Firefox.

    The doctype of this file claims to be "NETSCAPE-Bookmark-file-1".
    """
    bookmarks = []
    for e in root.find_all('dt', recursive=False):
        for link in e.find_all('a', recursive=False):
            bookmarks.append(firefox_link(
                link.get('href'),
                link.string,
                {attr: link.get(attr) for attr in link.attrs if attr != 'href'},
            ))

        for folder in e.find_all('h3', recursive=False):
            folder_name = folder.string
            folder_dl = folder.find_next_sibling('dl')
            bookmarks.append(bookmark_folder(
                folder_name,
                parse_bookmarks(folder_dl)
            ))
    return bookmarks


def parse_bookmarks_from_html(filename):
    with open(filename) as f:
        soup = BeautifulSoup(f, 'html5lib')
    return parse_bookmarks(soup.find('dl'))


def bookmark_addresses(bookmarks):
    """Generate addresses from a parsed tree of bookmarks."""
    # A bookmark is either a firefox_link or a bookmark_folder
    for bookmark in bookmarks:
        if 'address' in bookmark:
            yield bookmark['address']
        elif 'bookmarks' in bookmark:
            yield from bookmark_addresses(bookmark['bookmarks'])


def remove_duplicate_bookmarks(bookmarks, addresses):
    """Remove duplicate addresses from a tree of bookmarks."""
    out_bookmarks = []
    for bookmark in bookmarks:
        if 'address' in bookmark:
            if bookmark['address'] in addresses:
                print(f'Dupe: {bookmark["address"]}', file=sys.stderr)
            elif bookmark['address'].startswith('place:'):
                # Skip recents and other autogenerated bookmark lists
                continue
            else:
                out_bookmarks.append(bookmark)
        elif 'bookmarks' in bookmark:
            deduped = remove_duplicate_bookmarks(bookmark['bookmarks'], addresses)
            if deduped:
                out_bookmarks.append(bookmark_folder(bookmark['folder_name'], deduped))
    return out_bookmarks


@click.group(help=__doc__)
def cli():
    pass


@cli.command('extract')
@click.argument('bookmarks_filename')
def click_extract(bookmarks_filename):
    """Extract bookmarks in a bookmarks HTML file to structured JSON."""
    print(json.dumps(parse_bookmarks_from_html(bookmarks_filename), indent=4))


@cli.command('prepare-import')
@click.argument('primary_bookmarks_filename')
@click.argument('secondary_bookmarks_filename')
def click_merge(primary_bookmarks_filename, secondary_bookmarks_filename):
    """Prepare secondary bookmarks for import.

    Given an HTML bookmarks export from the primary Firefox profile and an HTML bookmarks export
    from the secondary Firefox profile, write an HTML file containing bookmarks that appear only in
    the secondary profile.

    The bookmark folder structure from the secondary file is preserved in a new folder with the
    following deduplication operations performed:

    - If a bookmark's address also appears in the primary file, it is omitted from the secondary.
    - If omissions of bookmarks result in empty folders, the empty folders are pruned.

    A new bookmarks file is written out, which can then be imported into the primary Firefox
    profile via the Bookmarks Manager.  The user can then use the Bookmarks Manager to manipulate
    the secondary's folder structure, merging it with the primary with the assurance that doing so
    won't result in adding duplicate bookmarks.
    """
    primary_bookmarks = parse_bookmarks_from_html(primary_bookmarks_filename)
    primary_addresses = set(bookmark_addresses(primary_bookmarks))

    secondary_bookmarks = parse_bookmarks_from_html(secondary_bookmarks_filename)

    deduplicated_secondary_bookmarks = remove_duplicate_bookmarks(
        secondary_bookmarks, primary_addresses)

    print(bookmarks_to_html(deduplicated_secondary_bookmarks))


if __name__ == '__main__':
    cli()

Extract and merge links from HTML bookmarks files exported from Firefox.  A Firefox bookmarks file doesn't make bookmarks available in a form that's easy to work with. This tool solves that problem. <!DOCTYPE NETSCAPE-Bookmark-file-1> <!-- This is an automatically generated file.      It will be read and overwritten.      DO NOT EDIT! --> <META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8"> <TITLE>Bookmarks</TITLE> <H1>Bookmarks Menu</H1>  <dl><p> # TODO the header also has ADD_DATE, LAST_MODIFIED, but I don't care much about those things. Parse the tree of bookmarks from an HTML bookmark file exported from Firefox.      The doctype of this file claims to be "NETSCAPE-Bookmark-file-1". Generate addresses from a parsed tree of bookmarks. # A bookmark is either a firefox_link or a bookmark_folder Remove duplicate addresses from a tree of bookmarks. # Skip recents and other autogenerated bookmark lists Extract bookmarks in a bookmarks HTML file to structured JSON. Prepare secondary bookmarks for import.      Given an HTML bookmarks export from the primary Firefox profile and an HTML bookmarks export     from the secondary Firefox profile, write an HTML file containing bookmarks that appear only in     the secondary profile.      The bookmark folder structure from the secondary file is preserved in a new folder with the     following deduplication operations performed:      - If a bookmark's address also appears in the primary file, it is omitted from the secondary.     - If omissions of bookmarks result in empty folders, the empty folders are pruned.      A new bookmarks file is written out, which can then be imported into the primary Firefox     profile via the Bookmarks Manager.  The user can then use the Bookmarks Manager to manipulate     the secondary's folder structure, merging it with the primary with the assurance that doing so     won't result in adding duplicate bookmarks.