Function makeunicodedata

Tools/unicode/makeunicodedata.py:174–491

(unicode, trace)

# unicode character properties

def makeunicodedata(unicode, trace):

    # the default value of east_asian_width is "N", for unassigned code points
    # not mentioned in EastAsianWidth.txt
    # in addition there are some reserved but unassigned code points in CJK
    # ranges that are classified as "W". code points in private use areas
    # have a width of "A". both of these have entries in
    # EastAsianWidth.txt
    # see https://unicode.org/reports/tr11/#Unassigned
    assert EASTASIANWIDTH_NAMES[0] == "N"
    assert GRAPHEME_CLUSTER_NAMES[0] == "Other"
    assert INDIC_CONJUNCT_BREAK_NAMES[0] == "None"
    dummy = (0, 0, 0, 0, 0, 0, 0, 0, 0)
    table = [dummy]
    cache = {0: dummy}
    index = [0] * len(unicode.chars)

    FILE = "Modules/unicodedata_db.h"

    print("--- Preparing", FILE, "...")

    # 1) database properties

    for char in unicode.chars:
        record = unicode.table[char]
        eastasianwidth = EASTASIANWIDTH_NAMES.index(unicode.widths[char] or 'N')
        graphemebreak = GRAPHEME_CLUSTER_NAMES.index(unicode.grapheme_breaks[char] or 'Other')
        extpict = unicode.ext_picts[char]
        bidirectional = BIDIRECTIONAL_NAMES.index(unicode.bidi_classes[char])
        if record:
            # extract database properties
            category = CATEGORY_NAMES.index(record.general_category)
            combining = int(record.canonical_combining_class)
            mirrored = record.bidi_mirrored == "Y"
            normalizationquickcheck = record.quick_check
            incb = INDIC_CONJUNCT_BREAK_NAMES.index(record.incb)
            item = (
                category, combining, bidirectional, mirrored, eastasianwidth,
                normalizationquickcheck, graphemebreak, incb, extpict,
                )
        else:
            if eastasianwidth or graphemebreak or extpict or bidirectional:
                item = (0, 0, bidirectional, 0, eastasianwidth,
                        0, graphemebreak, 0, extpict)
            else:
                continue

        # add entry to index and item tables
        i = cache.get(item)
        if i is None:
            cache[item] = i = len(table)
            table.append(item)
        index[char] = i

    # 2) decomposition data

    decomp_data_cache = {}
    decomp_data = [0]