Split the current Misc/NEWS into a zillion little blurb files.

Assumes that the newest version section in Misc/NEWS is under
development, and splits those entries into the "next" subdirectory.
If the current version has actually been released, use the
--released flag.

Also runs "blurb populate" for you.

def split(*, released=False):
    """
Split the current Misc/NEWS into a zillion little blurb files.

Assumes that the newest version section in Misc/NEWS is under
development, and splits those entries into the "next" subdirectory.
If the current version has actually been released, use the
--released flag.

Also runs "blurb populate" for you.
    """

    # note: populate also does chdir $python_root/Misc for you
    populate()

    if not os.path.isfile("NEWS"):
        error("You don't have a Misc/NEWS file!")

    def global_sections():
        global sections
        return sections

    sections = set(global_sections())
    release_date_marker = "Release date:"
    whats_new_marker = "What's New in Python "

    blurbs = Blurbs()

    accumulator = []
    bpo = "0"
    serial_number = 9999
    version = None
    release_date = None
    section = None
    see_also = None
    no_changes = None
    security = None
    blurb_count = 0
    version_count = 0


    def flush_blurb():
        nonlocal accumulator
        nonlocal serial_number
        nonlocal bpo
        nonlocal release_date
        nonlocal see_also
        nonlocal no_changes
        nonlocal blurb_count
        nonlocal security

        if accumulator:
            if version:
                # strip trailing blank lines
                while accumulator:
                    line = accumulator.pop()
                    if not line.rstrip():
                        continue
                    accumulator.append(line)
                    break
                if see_also:
                    fields = []
                    see_also = see_also.replace(" and ", ",")
                    for field in see_also.split(","):
                        field = field.strip()
                        if not field:
                            continue
                        if field.startswith("and "):
                            field = field[5:].lstrip()
                        if field.lower().startswith("issue"):
                            field = field[5:].strip()
                        if field.startswith("#"):
                            field = field[1:]
                        try:
                            int(field)
                            field = "bpo-" + field
                        except ValueError:
                            pass
                        fields.append(field)
                    see_also = ", ".join(fields)
                    # print("see_also: ", repr(see_also))
                    accumulator.append(f("(See also: {see_also})"))
                    see_also = None
                if not accumulator:
                    return
                if not (section or no_changes):
                    error("No section for line " + str(line_number) + "!")

                body = "\n".join(accumulator) + "\n"
                metadata = {}
                metadata["bpo"] = bpo
                metadata["date"] = str(serial_number)
                if section:
                    metadata["section"] = section
                else:
                    assert no_changes
                metadata["nonce"] = nonceify(body)
                if security:
                    # retroactively change section to "Security"
                    assert section
                    metadata["original section"] = metadata["section"]
                    metadata["section"] = "Security"

                if release_date is not None:
                    assert not len(blurbs)
                    metadata["release date"] = release_date
                    release_date = None
                if no_changes is not None:
                    assert not len(blurbs)
                    metadata["no changes"] = "True"
                    no_changes = None
                blurbs.append((metadata, body))
                blurb_count += 1

            bpo = "0"
            serial_number -= 1
            accumulator.clear()

    def flush_version():
        global git_add_files
        nonlocal released
        nonlocal version_count

        flush_blurb()
        if version is None:
            assert not blurbs, "version should only be None initially, we shouldn't have blurbs yet"
            return
        assert blurbs, f("No blurbs defined when flushing version {version}!")
        output = f("NEWS.d/{version}.rst")

        if released:
            # saving merged blurb file for version, e.g. Misc/NEWS.d/3.7.0a1.rst
            blurbs.save(output)
            git_add_files.append(output)
        else:
            # saving a million old-school blurb next files
            # with serial numbers instead of dates
            # e.g. Misc/NEWS.d/next/IDLE/094.bpo-25514.882pXa.rst
            filenames = blurbs.save_split_next()
            git_add_files.extend(filenames)
            released = True
        blurbs.clear()
        version_count += 1

    with open("NEWS", "rt", encoding="utf-8") as file:
        for line_number, line in enumerate(file):
            line = line.rstrip()

            if line.startswith("\ufeff"):
                line = line[1:]

            # clean the slightly dirty data:
            # 1. inconsistent names for sections, etc
            for old, new in (
                ("C-API", "C API"),
                ("Core and builtins", "Core and Builtins"),
                ("Tools", "Tools/Demos"),
                ("Tools / Demos", "Tools/Demos"),
                ("Misc", "Windows"), # only used twice, both were really Windows
                ("Mac", "macOS"),
                ("Mac OS X", "macOS"),
                ("Extension Modules", "Library"),
                ("Whats' New in Python 2.7.6?", "What's New in Python 2.7.6?"),
                ):
                if line == old:
                    line = new
            # 2. unusual indenting
            _line = line.lstrip()
            if _line.startswith(("- Issue #", "- bpo-")):
                line = _line
            if _line == ".characters() and ignorableWhitespace() methods.  Original patch by Sebastian":
                line = " " + line
            # 3. fix version for What's New
            if line.startswith(whats_new_marker):
                flush_version()
                version = line[len(whats_new_marker):].strip().lower()
                for old, new in (
                    ("?", ""),
                    (" alpha ", "a"),
                    (" beta ", "b"),
                    (" release candidate ", "rc"),
                    (" final", ""),
                    ("3.5a", "3.5.0a"),
                    ):
                    version = version.replace(old, new)
                section = None
                continue
            # 3.a. fix specific precious little snowflakes
            # who can't be bothered to follow our stifling style conventions
            # and like, did their own *thing*, man.
            if line.startswith("- Issue #27181 remove statistics.geometric_mean"):
                line = line.replace(" remove", ": remove")
            elif line.startswith("* bpo-30357: test_thread: setUp()"):
                line = line.replace("* bpo-30357", "- bpo-30357")
            elif line.startswith("- Issue #25262. Added support for BINBYTES8"):
                line = line.replace("#25262.", "#25262:")
            elif line.startswith("- Issue #21032. Fixed socket leak if"):
                line = line.replace("#21032.", "#21032:")
            elif line.startswith("- Issue ##665194: Update "):
                line = line.replace("##665194", "#665194")
            elif line.startswith("- Issue #13449 sched.scheduler.run()"):
                line = line.replace("#13449 sched", "#13449: sched")
            elif line.startswith("- Issue #8684 sched.scheduler class"):
                line = line.replace("#8684 sched", "#8684: sched")
            elif line.startswith(" bpo-29243: Prevent unnecessary rebuilding"):
                line = line.replace(" bpo-29243:", "- bpo-29243:")
            elif line.startswith((
                "- Issue #11603 (again): Setting",
                "- Issue #15801 (again): With string",
                )):
                line = line.replace(" (again):", ":")
            elif line.startswith("- Issue #1665206 (partially): "):
                line = line.replace(" (partially):", ":")
            elif line.startswith("- Issue #2885 (partial): The"):
                line = line.replace(" (partial):", ":")
            elif line.startswith("- Issue #2885 (partial): The"):
                line = line.replace(" (partial):", ":")
            elif line.startswith("- Issue #1797 (partial fix):"):
                line = line.replace(" (partial fix):", ":")
            elif line.startswith("- Issue #5828 (Invalid behavior of unicode.lower): Fixed bogus logic in"):
                line = line.replace(" (Invalid behavior of unicode.lower):", ":")
            elif line.startswith("- Issue #4512 (part 2): Promote ``ZipImporter._get_filename()`` to be a public"):
                line = line.replace(" (part 2):", ":")
            elif line.startswith("- Revert bpo-26293 for zipfile breakage. See also bpo-29094."):
                line = "- bpo-26293, bpo-29094: Change resulted because of zipfile breakage."
            elif line.startswith("- Revert a37cc3d926ec (Issue #5322)."):
                line = "- Issue #5322: Revert a37cc3d926ec."
            elif line.startswith("- Patch #1970 by Antoine Pitrou: Speedup unicode whitespace and"):
                line = "- Issue #1970: Speedup unicode whitespace and"
            elif line.startswith("  linebreak detection"):
                line = "  linebreak detection.  (Patch by Antoine Pitrou.)"
            elif line.startswith("- Patch #1182394 from Shane Holloway: speed up HMAC.hexdigest."):
                line = "- Issue #1182394: Speed up ``HMAC.hexdigest``.  (Patch by Shane Holloway.)"
            elif line.startswith("- Variant of patch #697613: don't exit the interpreter on a SystemExit"):
                line = "- Issue #697613: Don't exit the interpreter on a SystemExit"
            elif line.startswith("- Bugs #1668596/#1720897: distutils now copies data files even if"):
                line = "- Issue #1668596, #1720897: distutils now copies data files even if"
            elif line.startswith("- Reverted patch #1504333 to sgmllib because it introduced an infinite"):
                line = "- Issue #1504333: Reverted change to sgmllib because it introduced an infinite"
            elif line.startswith("- PEP 465 and Issue #21176: Add the '@' operator for matrix multiplication."):
                line = "- Issue #21176: PEP 465: Add the '@' operator for matrix multiplication."
            elif line.startswith("- Issue: #15138: base64.urlsafe_{en,de}code() are now 3-4x faster."):
                line = "- Issue #15138: base64.urlsafe_{en,de}code() are now 3-4x faster."
            elif line.startswith("- Issue #9516: Issue #9516: avoid errors in sysconfig when MACOSX_DEPLOYMENT_TARGET"):
                line = "- Issue #9516 and Issue #9516: avoid errors in sysconfig when MACOSX_DEPLOYMENT_TARGET"
            elif line.title().startswith(("- Request #", "- Bug #", "- Patch #", "- Patches #")):
                # print(f("FIXING LINE {line_number}: {line!r}"))
                line = "- Issue #" + line.partition('#')[2]
                # print(f("FIXED LINE {line_number}: {line!r}"))
            # else:
            #     print(f("NOT FIXING LINE {line_number}: {line!r}"))


            # 4. determine the actual content of the line

            # 4.1 section declaration
            if line in sections:
                flush_blurb()
                section = line
                continue

            # 4.2 heading ReST marker
            if line.startswith((
                "===",
                "---",
                "---",
                "+++",
                "Python News",
                "**(For information about older versions, consult the HISTORY file.)**",
                )):
                continue

            # 4.3 version release date declaration
            if line.startswith(release_date_marker) or (
                line.startswith("*") and release_date_marker in line):
                while line.startswith("*"):
                    line = line[1:]
                while line.endswith("*"):
                    line = line[:-1]
                release_date = line[len(release_date_marker):].strip()
                continue

            # 4.4 no changes declaration
            if line.strip() in (
                '- No changes since release candidate 2',
                'No changes from release candidate 2.',
                'There were no code changes between 3.5.3rc1 and 3.5.3 final.',
                'There were no changes between 3.4.6rc1 and 3.4.6 final.',
                ):
                no_changes = True
                if line.startswith("- "):
                    line = line[2:]
                accumulator.append(line)
                continue

            # 4.5 start of new blurb
            if line.startswith("- "):
                flush_blurb()
                line = line[2:]
                security = line.startswith("[Security]")
                if security:
                    line = line[10:].lstrip()

                if line.startswith("Issue"):
                    line = line[5:].lstrip()
                    if line.startswith("s"):
                        line = line[1:]
                    line = line.lstrip()
                    if line.startswith("#"):
                        line = line[1:].lstrip()
                    parse_bpo = True
                elif line.startswith("bpo-"):
                    line = line[4:]
                    parse_bpo = True
                else:
                    # print(f("[[{line_number:8} no bpo]] {line}"))
                    parse_bpo = False
                if parse_bpo:
                    # GAAAH
                    if line == "17500, and https://github.com/python/pythondotorg/issues/945: Remove":
                        line = "Remove"
                        bpo = "17500"
                        see_also = "https://github.com/python/pythondotorg/issues/945"
                    else:
                        bpo, colon, line = line.partition(":")
                        line = line.lstrip()
                        bpo, comma, see_also = bpo.partition(",")
                        if comma:
                            see_also = see_also.strip()
                            # if it's just an integer, add bpo- to the front
                            try:
                                int(see_also)
                                see_also = "bpo-" + see_also
                            except ValueError:
                                pass
                        else:
                            # - Issue #21529 (CVE-2014-4616)
                            bpo, space_paren, see_also = bpo.partition(" (")
                            if space_paren:
                                see_also = see_also.rstrip(")")
                            else:
                                # - Issue #19544 and Issue #1180:
                                bpo, space_and, see_also = bpo.partition(" and ")
                                if not space_and:
                                    bpo, space_and, see_also = bpo.partition(" & ")
                                if space_and:
                                    see_also = see_also.replace("Issue #", "bpo-").strip()
                                else:
                                    # - Issue #5258/#10642: if site.py
                                    bpo, slash, see_also = bpo.partition("/")
                                    if space_and:
                                        see_also = see_also.replace("#", "bpo-").strip()
                    try:
                        int(bpo) # this will throw if it's not a legal int
                    except ValueError:
                        sys.exit(f("Couldn't convert bpo number to int on line {line_number}! {bpo!r}"))
                    if see_also == "partially":
                        sys.exit(f("What the hell on line {line_number}! {bpo!r}"))

            # 4.6.1 continuation of blurb
            elif line.startswith("  "):
                line = line[2:]
            # 4.6.2 continuation of blurb
            elif line.startswith(" * "):
                line = line[3:]
            elif line:
                sys.exit(f("Didn't recognize line {line_number}! {line!r}"))
            # only add blank lines if we have an initial line in the accumulator
            if line or accumulator:
                accumulator.append(line)

    flush_version()

    assert git_add_files
    flush_git_add_files()
    git_rm_files.append("NEWS")
    flush_git_rm_files()

    print(f("Wrote {blurb_count} news items across {version_count} versions."))
    print()
    print("Ready for commit.")