<filename>utils/genUnicodeTable.py
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the LICENSE
# file in the root directory of this source tree.

# -*- coding: utf-8 -*-

# Generates list of unicode ranges belonging to a set of categories
# Usage: genUnicodeTable.py

import datetime
import hashlib
import sys
import urllib.request
from string import Template


UNICODE_DATA_URL = "ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt"

UNICODE_SPECIAL_CASING_URL = "ftp://ftp.unicode.org/Public/UNIDATA/SpecialCasing.txt"


# Unicode data field indexes. See UnicodeData.txt.
CODEPOINT_FIELD = 0
GENERAL_CATEGORY_FIELD = 2
UPPERCASE_FIELD = 12
LOWERCASE_FIELD = 13


def from_hexpoints(s):
    """Parses a string from its codepoints as hex.

    Given a string containing one or more code points (as hex values),
    parse it to a a string. For example, the input "006E 006F" produces
    output "no".

    """
    return "".join([chr(int(cp, 16)) for cp in s.split()])


def print_header(unicodedata_sha1, specialcasing_sha1):
    print("//")
    print("// File generated by genUnicodeTable.py")
    print(
        "// using UnicodeData.txt and SpecialCasing.txt downloaded on "
        + str(datetime.date.today())
    )
    print("// UnicodeData.txt SHA1:   " + unicodedata_sha1)
    print("// SpecialCasing.txt SHA1: " + specialcasing_sha1)
    print("// *** DO NOT EDIT BY HAND ***")
    print("//")
    print("")


def run_interval(unicode_data_lines, args):
    name = args[0]
    categories = set(args[1:])
    begin = 0
    intervals = []
    last_cp = 0
    openi = False
    for line in unicode_data_lines:
        fields = line.split(";")
        cp_str, category = fields[CODEPOINT_FIELD], fields[GENERAL_CATEGORY_FIELD]
        cp = ord(from_hexpoints(cp_str))
        if category in categories:
            if not openi:
                begin = cp
                openi = True
            else:
                pass  # do nothing we are still in interval
        else:
            if openi:
                intervals.append((begin, last_cp))
                openi = False
            else:
                pass  # keep looking
        last_cp = cp

    if openi:
        intervals.append((begin, last_cp))

    print("// " + " ".join(args))
    sizename = name + "_SIZE"
    print(
        "// static constexpr uint32_t " + sizename + " = " + str(len(intervals)) + ";"
    )
    print("static constexpr UnicodeRange " + name + "[] = {")
    for i in intervals:
        print("{" + hex(i[0]) + ", " + hex(i[1]) + "},")
    print("};")
    print("")


def print_categories(unicode_data_lines):
    """Output UnicodeRanges for Unicode General Categories."""
    print("struct UnicodeRange { uint32_t first; uint32_t second; };")
    print("")
    categories = [
        "UNICODE_LETTERS Lu Ll Lt Lm Lo Nl",
        "UNICODE_COMBINING_MARK Mn Mc",
        "UNICODE_DIGIT Nd",
        "UNICODE_CONNECTOR_PUNCTUATION Pc",
    ]
    for cat in categories:
        run_interval(unicode_data_lines, cat.split())


class CaseMap(object):
    """Unicode case mapping helper.

    This class holds the list of codepoints, and their uppercase and
    lowercase mappings.

    """

    def __init__(self, unicode_data_lines, special_casing_lines):
        """Construct with the lines from UnicodeData and SpecialCasing."""
        self.toupper = {}
        self.tolower = {}
        self.codepoints = []
        for line in unicode_data_lines:
            fields = line.split(";")
            self.__set_casemap(
                fields[CODEPOINT_FIELD],
                upper=fields[UPPERCASE_FIELD],
                lower=fields[LOWERCASE_FIELD],
            )
        self.codepoints.extend(self.toupper.keys())

        # Apply special cases. This is to support ES5.1 Canonicalize, which is
        # cast in terms of toUpperCase(). The desire here is to have a
        # locale-independent result. Thus we ignore SpecialCasing rules that
        # are locale specific. We can also get away with ignoring
        # context-sensitive rules because Canonicalize only considers one
        # character. Thus ignore any rules that have a condition.
        # Format is codepoint, lower, title, upper, condition
        for line in special_casing_lines:
            # Trim comments
            line = line.split("#")[0]
            fields = line.split(";")
            if len(fields) < 5:
                continue
            cps, lower, title, upper, condition = fields[:5]
            if not condition.strip():
                self.__set_casemap(cps, upper=upper, lower=lower)

    def __set_casemap(self, cp, upper, lower):
        """Set a case mapping.

        Mark the upper and lower case forms of cp. If a form is empty,
        the character is its own case mapping. All strings are encoded
        as hexpoints (e.g. '000F')

        """
        cp = from_hexpoints(cp)
        # "The simple uppercase is omitted in the data file if the uppercase
        # is the same as the code point itself."
        # The same is true for the lowercase.
        self.toupper[cp] = from_hexpoints(upper) if upper else cp
        self.tolower[cp] = from_hexpoints(lower) if lower else cp

    def canonicalize(self, ch):
        """Canonicalize a character per ES5.1 15.10.2.8."""
        upper_ch = self.toupper[ch]
        # "If u does not consist of a single character, return ch"
        if len(upper_ch) != 1:
            return ch
        # "If ch's code unit value is greater than or equal to decimal 128
        # and cu's code unit value is less than decimal 128, then return ch"
        # That is, only ASCII may canonicalize to ASCII.
        if ord(upper_ch) < 128 and ord(ch) >= 128:
            return ch
        return upper_ch


def print_precanonicalizations(casemap):
    """Print a table of pre-canonicalizations.

    For each canonicalized code point, print the list of code points
    that canonicalize to it (its "pre-canonicalizations"). However do
    not print entries for characters whose pre-canonicalizations are
    only the character and its lowercase form.

    """
    # Get the simple case mappings, then build an inverted table for
    # canonicalizations.
    precanons = {}
    for cp in casemap.codepoints:
        canon_cp = casemap.canonicalize(cp)
        precanons.setdefault(canon_cp, []).append(cp)

    # Remove "obvious" entries that consist of exactly the character
    # and its lowercase form.
    for canon_cp in list(precanons.keys()):
        trivial_precanons = {canon_cp, casemap.tolower[canon_cp]}
        if set(precanons[canon_cp]) == trivial_precanons:
            del precanons[canon_cp]

    # Construct the entries.
    # Each entry leads with the canonicalized codepoint, then the codepoints
    # that canonicalize to it. Example: {0x1c4, {0x1c4, 0x1c5, 0x1c6}}
    def as_hex(s):
        return " ".join(["0x%.4X" % ord(c) for c in s])

    entries = []
    for canon_cp in sorted(precanons.keys()):
        cps_strs = [as_hex(cp) for cp in precanons[canon_cp] if cp != canon_cp]
        entries.append("{%s, {%s}}" % (as_hex(canon_cp), ", ".join(cps_strs)))

    # Print the table.
    code = Template(
        """
struct UnicodePrecanonicalizationMapping {
    /// The canonicalized form of the character.
    uint16_t canonicalized;

    /// A list of up to 3 characters which canonicalize to this character.
    /// The value 3 is significant because it is the maximum number of
    /// pre-canonicalizations of any character.
    /// 0 (NUL) is used to indicate none.
    uint16_t forms[3];
};

// The precanonicalizations is a list of exceptional canocializations.
// That is, each canonicalized input character maps to a list of forms that
// canonicalize to it, per the algorithm given in ES5 15.10.2.8. However, if a
// character is only canonicalized to by itself and its lowercase variant, that
// is omitted from the table; this helps keep the table small. Note some
// entries are empty; this indicates that c != uppercase(lowercase(c)). Note
// also this table is sorted.
static constexpr uint32_t UNICODE_PRECANONS_SIZE = $entry_count;
static constexpr UnicodePrecanonicalizationMapping UNICODE_PRECANONS[] = {
$entries
};
    """
    ).substitute(entry_count=len(entries), entries=",\n".join(entries))
    print(code.strip())


if __name__ == "__main__":
    print("Fetching %s..." % UNICODE_DATA_URL, file=sys.stderr)
    with urllib.request.urlopen(UNICODE_DATA_URL) as f:
        unicode_data = f.read()

    print("Fetching %s..." % UNICODE_SPECIAL_CASING_URL, file=sys.stderr)
    with urllib.request.urlopen(UNICODE_SPECIAL_CASING_URL) as f:
        special_casing = f.read()

    print_header(
        hashlib.sha1(unicode_data).hexdigest(), hashlib.sha1(special_casing).hexdigest()
    )
    udata_lines = unicode_data.decode("utf-8").splitlines()
    special_lines = special_casing.decode("utf-8").splitlines()
    print_categories(udata_lines)
    print_precanonicalizations(CaseMap(udata_lines, special_lines))

<filename>utils/genUnicodeTable.py
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the LICENSE
# file in the root directory of this source tree.

# -*- coding: utf-8 -*-

# Generates list of unicode ranges belonging to a set of categories
# Usage: genUnicodeTable.py

import datetime
import hashlib
import sys
import urllib.request
from string import Template


UNICODE_DATA_URL = "ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt"

UNICODE_SPECIAL_CASING_URL = "ftp://ftp.unicode.org/Public/UNIDATA/SpecialCasing.txt"


# Unicode data field indexes. See UnicodeData.txt.
CODEPOINT_FIELD = 0
GENERAL_CATEGORY_FIELD = 2
UPPERCASE_FIELD = 12
LOWERCASE_FIELD = 13


def from_hexpoints(s):
    """Parses a string from its codepoints as hex.

    Given a string containing one or more code points (as hex values),
    parse it to a a string. For example, the input "006E 006F" produces
    output "no".

    """
    return "".join([chr(int(cp, 16)) for cp in s.split()])


def print_header(unicodedata_sha1, specialcasing_sha1):
    print("//")
    print("// File generated by genUnicodeTable.py")
    print(
        "// using UnicodeData.txt and SpecialCasing.txt downloaded on "
        + str(datetime.date.today())
    )
    print("// UnicodeData.txt SHA1:   " + unicodedata_sha1)
    print("// SpecialCasing.txt SHA1: " + specialcasing_sha1)
    print("// *** DO NOT EDIT BY HAND ***")
    print("//")
    print("")


def run_interval(unicode_data_lines, args):
    name = args[0]
    categories = set(args[1:])
    begin = 0
    intervals = []
    last_cp = 0
    openi = False
    for line in unicode_data_lines:
        fields = line.split(";")
        cp_str, category = fields[CODEPOINT_FIELD], fields[GENERAL_CATEGORY_FIELD]
        cp = ord(from_hexpoints(cp_str))
        if category in categories:
            if not openi:
                begin = cp
                openi = True
            else:
                pass  # do nothing we are still in interval
        else:
            if openi:
                intervals.append((begin, last_cp))
                openi = False
            else:
                pass  # keep looking
        last_cp = cp

    if openi:
        intervals.append((begin, last_cp))

    print("// " + " ".join(args))
    sizename = name + "_SIZE"
    print(
        "// static constexpr uint32_t " + sizename + " = " + str(len(intervals)) + ";"
    )
    print("static constexpr UnicodeRange " + name + "[] = {")
    for i in intervals:
        print("{" + hex(i[0]) + ", " + hex(i[1]) + "},")
    print("};")
    print("")


def print_categories(unicode_data_lines):
    """Output UnicodeRanges for Unicode General Categories."""
    print("struct UnicodeRange { uint32_t first; uint32_t second; };")
    print("")
    categories = [
        "UNICODE_LETTERS Lu Ll Lt Lm Lo Nl",
        "UNICODE_COMBINING_MARK Mn Mc",
        "UNICODE_DIGIT Nd",
        "UNICODE_CONNECTOR_PUNCTUATION Pc",
    ]
    for cat in categories:
        run_interval(unicode_data_lines, cat.split())


class CaseMap(object):
    """Unicode case mapping helper.

    This class holds the list of codepoints, and their uppercase and
    lowercase mappings.

    """

    def __init__(self, unicode_data_lines, special_casing_lines):
        """Construct with the lines from UnicodeData and SpecialCasing."""
        self.toupper = {}
        self.tolower = {}
        self.codepoints = []
        for line in unicode_data_lines:
            fields = line.split(";")
            self.__set_casemap(
                fields[CODEPOINT_FIELD],
                upper=fields[UPPERCASE_FIELD],
                lower=fields[LOWERCASE_FIELD],
            )
        self.codepoints.extend(self.toupper.keys())

        # Apply special cases. This is to support ES5.1 Canonicalize, which is
        # cast in terms of toUpperCase(). The desire here is to have a
        # locale-independent result. Thus we ignore SpecialCasing rules that
        # are locale specific. We can also get away with ignoring
        # context-sensitive rules because Canonicalize only considers one
        # character. Thus ignore any rules that have a condition.
        # Format is codepoint, lower, title, upper, condition
        for line in special_casing_lines:
            # Trim comments
            line = line.split("#")[0]
            fields = line.split(";")
            if len(fields) < 5:
                continue
            cps, lower, title, upper, condition = fields[:5]
            if not condition.strip():
                self.__set_casemap(cps, upper=upper, lower=lower)

    def __set_casemap(self, cp, upper, lower):
        """Set a case mapping.

        Mark the upper and lower case forms of cp. If a form is empty,
        the character is its own case mapping. All strings are encoded
        as hexpoints (e.g. '000F')

        """
        cp = from_hexpoints(cp)
        # "The simple uppercase is omitted in the data file if the uppercase
        # is the same as the code point itself."
        # The same is true for the lowercase.
        self.toupper[cp] = from_hexpoints(upper) if upper else cp
        self.tolower[cp] = from_hexpoints(lower) if lower else cp

    def canonicalize(self, ch):
        """Canonicalize a character per ES5.1 15.10.2.8."""
        upper_ch = self.toupper[ch]
        # "If u does not consist of a single character, return ch"
        if len(upper_ch) != 1:
            return ch
        # "If ch's code unit value is greater than or equal to decimal 128
        # and cu's code unit value is less than decimal 128, then return ch"
        # That is, only ASCII may canonicalize to ASCII.
        if ord(upper_ch) < 128 and ord(ch) >= 128:
            return ch
        return upper_ch


def print_precanonicalizations(casemap):
    """Print a table of pre-canonicalizations.

    For each canonicalized code point, print the list of code points
    that canonicalize to it (its "pre-canonicalizations"). However do
    not print entries for characters whose pre-canonicalizations are
    only the character and its lowercase form.

    """
    # Get the simple case mappings, then build an inverted table for
    # canonicalizations.
    precanons = {}
    for cp in casemap.codepoints:
        canon_cp = casemap.canonicalize(cp)
        precanons.setdefault(canon_cp, []).append(cp)

    # Remove "obvious" entries that consist of exactly the character
    # and its lowercase form.
    for canon_cp in list(precanons.keys()):
        trivial_precanons = {canon_cp, casemap.tolower[canon_cp]}
        if set(precanons[canon_cp]) == trivial_precanons:
            del precanons[canon_cp]

    # Construct the entries.
    # Each entry leads with the canonicalized codepoint, then the codepoints
    # that canonicalize to it. Example: {0x1c4, {0x1c4, 0x1c5, 0x1c6}}
    def as_hex(s):
        return " ".join(["0x%.4X" % ord(c) for c in s])

    entries = []
    for canon_cp in sorted(precanons.keys()):
        cps_strs = [as_hex(cp) for cp in precanons[canon_cp] if cp != canon_cp]
        entries.append("{%s, {%s}}" % (as_hex(canon_cp), ", ".join(cps_strs)))

    # Print the table.
    code = Template(
        """
struct UnicodePrecanonicalizationMapping {
    /// The canonicalized form of the character.
    uint16_t canonicalized;

    /// A list of up to 3 characters which canonicalize to this character.
    /// The value 3 is significant because it is the maximum number of
    /// pre-canonicalizations of any character.
    /// 0 (NUL) is used to indicate none.
    uint16_t forms[3];
};

// The precanonicalizations is a list of exceptional canocializations.
// That is, each canonicalized input character maps to a list of forms that
// canonicalize to it, per the algorithm given in ES5 15.10.2.8. However, if a
// character is only canonicalized to by itself and its lowercase variant, that
// is omitted from the table; this helps keep the table small. Note some
// entries are empty; this indicates that c != uppercase(lowercase(c)). Note
// also this table is sorted.
static constexpr uint32_t UNICODE_PRECANONS_SIZE = $entry_count;
static constexpr UnicodePrecanonicalizationMapping UNICODE_PRECANONS[] = {
$entries
};
    """
    ).substitute(entry_count=len(entries), entries=",\n".join(entries))
    print(code.strip())


if __name__ == "__main__":
    print("Fetching %s..." % UNICODE_DATA_URL, file=sys.stderr)
    with urllib.request.urlopen(UNICODE_DATA_URL) as f:
        unicode_data = f.read()

    print("Fetching %s..." % UNICODE_SPECIAL_CASING_URL, file=sys.stderr)
    with urllib.request.urlopen(UNICODE_SPECIAL_CASING_URL) as f:
        special_casing = f.read()

    print_header(
        hashlib.sha1(unicode_data).hexdigest(), hashlib.sha1(special_casing).hexdigest()
    )
    udata_lines = unicode_data.decode("utf-8").splitlines()
    special_lines = special_casing.decode("utf-8").splitlines()
    print_categories(udata_lines)
    print_precanonicalizations(CaseMap(udata_lines, special_lines))

#!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the LICENSE # file in the root directory of this source tree. # -*- coding: utf-8 -*- # Generates list of unicode ranges belonging to a set of categories # Usage: genUnicodeTable.py # Unicode data field indexes. See UnicodeData.txt. Parses a string from its codepoints as hex.      Given a string containing one or more code points (as hex values),     parse it to a a string. For example, the input "006E 006F" produces     output "no". # do nothing we are still in interval # keep looking Output UnicodeRanges for Unicode General Categories. Unicode case mapping helper.      This class holds the list of codepoints, and their uppercase and     lowercase mappings. Construct with the lines from UnicodeData and SpecialCasing. # Apply special cases. This is to support ES5.1 Canonicalize, which is # cast in terms of toUpperCase(). The desire here is to have a # locale-independent result. Thus we ignore SpecialCasing rules that # are locale specific. We can also get away with ignoring # context-sensitive rules because Canonicalize only considers one # character. Thus ignore any rules that have a condition. # Format is codepoint, lower, title, upper, condition # Trim comments Set a case mapping.          Mark the upper and lower case forms of cp. If a form is empty,         the character is its own case mapping. All strings are encoded         as hexpoints (e.g. '000F') # "The simple uppercase is omitted in the data file if the uppercase # is the same as the code point itself." # The same is true for the lowercase. Canonicalize a character per ES5.1 15.10.2.8. # "If u does not consist of a single character, return ch" # "If ch's code unit value is greater than or equal to decimal 128 # and cu's code unit value is less than decimal 128, then return ch" # That is, only ASCII may canonicalize to ASCII. Print a table of pre-canonicalizations.      For each canonicalized code point, print the list of code points     that canonicalize to it (its "pre-canonicalizations"). However do     not print entries for characters whose pre-canonicalizations are     only the character and its lowercase form. # Get the simple case mappings, then build an inverted table for # canonicalizations. # Remove "obvious" entries that consist of exactly the character # and its lowercase form. # Construct the entries. # Each entry leads with the canonicalized codepoint, then the codepoints # that canonicalize to it. Example: {0x1c4, {0x1c4, 0x1c5, 0x1c6}} # Print the table. struct UnicodePrecanonicalizationMapping {     /// The canonicalized form of the character.     uint16_t canonicalized;      /// A list of up to 3 characters which canonicalize to this character.     /// The value 3 is significant because it is the maximum number of     /// pre-canonicalizations of any character.     /// 0 (NUL) is used to indicate none.     uint16_t forms[3]; };  // The precanonicalizations is a list of exceptional canocializations. // That is, each canonicalized input character maps to a list of forms that // canonicalize to it, per the algorithm given in ES5 15.10.2.8. However, if a // character is only canonicalized to by itself and its lowercase variant, that // is omitted from the table; this helps keep the table small. Note some // entries are empty; this indicates that c != uppercase(lowercase(c)). Note // also this table is sorted. static constexpr uint32_t UNICODE_PRECANONS_SIZE = $entry_count; static constexpr UnicodePrecanonicalizationMapping UNICODE_PRECANONS[] = { $entries };