<reponame>The-Orizon/nlputils
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import itertools

'''
Various functions for processing Chinese text.
'''

halfwidth = frozenset('!(),:;?')
fullwidth = frozenset(itertools.chain(
    range(0xFF02, 0xFF07 + 1),
    (0xFF0A, 0xFF0B, 0xFF0E, 0xFF0F, 0xFF1C, 0xFF1D,
     0xFF1E, 0xFF3C, 0xFF3E, 0xFF3F, 0xFF40),
    range(0xFF10, 0xFF19 + 1),
    range(0xFF20, 0xFF3A + 1),
    range(0xFF41, 0xFF5A + 1)))
resentencesp = re.compile('([﹒﹔﹖﹗．；。！？]["’”」』]{0,2}|：(?=["‘“「『]{1,2}|$))')
refixmissing = re.compile(
    '(^[^"‘“「『’”」』，；。！？]+["’”」』]|^["‘“「『]?[^"‘“「『’”」』]+[，；。！？][^"‘“「『‘“「『]*["’”」』])(?!["‘“「『’”」』，；。！？])')

punctstr = (
    '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¢£¥·ˇˉ―‖‘’“”•′‵、。々'
    '〈〉《》「」『』【】〔〕〖〗〝〞︰︱︳︴︵︶︷︸︹︺︻︼︽︾︿﹀﹁﹂﹃﹄'
    '﹏﹐﹒﹔﹕﹖﹗﹙﹚﹛﹜﹝﹞！（），．：；？［｛｜｝～､￠￡￥')

punct = frozenset(punctstr)

whitespace = ' \t\n\r\x0b\x0c\u3000'

resplitpunct = re.compile('([%s])' % re.escape(punctstr))

tailpunct = ('''!),-.:;?]}¢·ˇˉ―‖’”•′■□△○●'''
             '''、。々〉》」』】〕〗〞︰︱︳︴︶︸︺︼︾﹀﹂﹄﹏'''
             '''﹐﹒﹔﹕﹖﹗﹚﹜﹞！），．：；？｜］｝～､￠''') + whitespace
headpunct = ('''([`{£¥‘“〈《「『【〔〖〝'''
             '''︵︷︹︻︽︿﹁﹃﹙﹛﹝（［｛￡￥''') + whitespace

openbrckt = ('([{（［｛⦅〚⦃“‘‹«「〈《【〔⦗『〖〘｢⟦⟨⟪⟮⟬⌈⌊⦇⦉❛❝❨❪❴❬❮❰❲'
             '⏜⎴⏞〝︵⏠﹁﹃︹︻︗︿︽﹇︷〈⦑⧼﹙﹛﹝⁽₍⦋⦍⦏⁅⸢⸤⟅⦓⦕⸦⸨｟⧘⧚⸜⸌⸂⸄⸉᚛༺༼')
clozbrckt = (')]}）］｝⦆〛⦄”’›»」〉》】〕⦘』〗〙｣⟧⟩⟫⟯⟭⌉⌋⦈⦊❜❞❩❫❵❭❯❱❳'
             '⏝⎵⏟〞︶⏡﹂﹄︺︼︘﹀︾﹈︸〉⦒⧽﹚﹜﹞⁾₎⦌⦎⦐⁆⸣⸥⟆⦔⦖⸧⸩｠⧙⧛⸝⸍⸃⸅⸊᚜༻༽')

ucjk = frozenset(itertools.chain(
    range(0x1100, 0x11FF + 1),
    range(0x2E80, 0xA4CF + 1),
    range(0xA840, 0xA87F + 1),
    range(0xAC00, 0xD7AF + 1),
    range(0xF900, 0xFAFF + 1),
    range(0xFE30, 0xFE4F + 1),
    range(0xFF65, 0xFFDC + 1),
    range(0xFF01, 0xFF0F + 1),
    range(0xFF1A, 0xFF20 + 1),
    range(0xFF3B, 0xFF40 + 1),
    range(0xFF5B, 0xFF60 + 1),
    range(0x20000, 0x2FFFF + 1)
))

zhmodel = None
_curpath = os.path.normpath(
    os.path.join(os.getcwd(), os.path.dirname(__file__)))

RE_WS_IN_FW = re.compile(
    r'([‘’“”…─\u2e80-\u312f\u3200-\u32ff\u3400-\u4dbf\u4e00-\u9fff'
    '\uf900-\ufaff\ufe30-\ufe57\uff00-\uffef\U00020000-\U0002A6DF'
    '\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF'
    '\U0002F800-\U0002FA1F])\s+(?=[‘’“”…\u2e80-\u312f\u3200-\u32ff'
    '\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\ufe30-\ufe57\uff00-\uffef'
    '\U00020000-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F'
    '\U0002B820-\U0002CEAF\U0002F800-\U0002FA1F])')

RE_FW = re.compile(
    '([\u2018\u2019\u201c\u201d\u2026\u2e80-\u312f\u3200-\u32ff'
    '\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\ufe30-\ufe57\uff00-\uffef'
    '\U00020000-\U0002A6DF\U0002A700-\U0002B73F'
    '\U0002B740-\U0002B81F\U0002B820-\U0002CEAF'
    '\U0002F800-\U0002FA1F]+)')

RE_UCJK = re.compile(
    '([\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff'
    '\U00020000-\U0002A6DF\U0002A700-\U0002B73F'
    '\U0002B740-\U0002B81F\U0002B820-\U0002CEAF'
    '\U0002F800-\U0002FA1F]+)'
)

RE_CTRL = re.compile("[\000-\037\ufeff]+")

hasucjk = lambda s: RE_UCJK.search(s)
removectrl = lambda s: RE_CTRL.sub('', s)

detokenize = lambda s: RE_WS_IN_FW.sub(r'\1', s).strip()
detokenize.__doc__ = 'Detokenization function for Chinese.'

def splitsentence(sentence):
    '''Split a piece of Chinese into sentences.'''
    # s = ''.join((chr(ord(ch)+0xFEE0) if ch in halfwidth else ch) for ch in sentence)
    s = sentence
    slist = []
    for i in resentencesp.split(s):
        if resentencesp.match(i) and slist:
            slist[-1] += i
        elif i:
            slist.append(i)
    return slist


def splithard(sentence, maxchar=None):
    '''Forcely split a piece of Chinese into sentences with the limit of max sentence length.'''
    slist = splitsentence(sentence)
    if maxchar is None:
        return slist
    slist1 = []
    for sent in slist:
        if len(sent) > maxchar:
            for i in resplitpunct.split(sent):
                if resplitpunct.match(i) and slist1:
                    slist1[-1] += i
                elif i:
                    slist1.append(i)
        else:
            slist1.append(sent)
    slist = slist1
    slist1 = []
    for sent in slist:
        if len(sent) > maxchar:
            slist1.extend(sent[i:i + maxchar]
                          for i in range(0, len(sent), maxchar))
        else:
            slist1.append(sent)
    slist = slist1
    return slist


def fixmissing(slist):
    '''Fix missing quotes.'''
    newlist = []
    for i in slist:
        newlist.extend(filter(None, refixmissing.split(i)))
    return newlist


def filterlist(slist):
    '''Get meaningful sentences.'''
    for i in slist:
        s = i.lstrip(tailpunct).rstrip(headpunct)
        if len(s) > 1:
            yield s


def addwalls(tokiter):
    '''Add walls between punctuations for Moses.'''
    lastwall = False
    for tok in tokiter:
        if tok in punct:
            if not lastwall:
                yield '<wall />'
            yield tok
            yield '<wall />'
            lastwall = True
        else:
            yield tok
            lastwall = False


def addwallzone(tokiter):
    '''Add walls and zones between punctuations for Moses.'''
    W = '<wall />'
    out = []
    expect = zidx = None
    for tok in tokiter:
        if tok in punct:
            if not (out and out[-1] == W):
                out.append(W)
            if tok == expect:
                out[zidx] = '<zone>'
                out.append(tok)
                out.append('</zone>')
                expect = zidx = None
            else:
                bid = openbrckt.find(tok)
                if bid > -1:
                    expect = clozbrckt[bid]
                    zidx = len(out) - 1
                out.append(tok)
                out.append(W)
        else:
            out.append(tok)
    if out and out[0] == W:
        out.pop(0)
    if out and out[-1] == W:
        out.pop()
    return out


def calctxtstat(s):
    '''Detect whether a string is modern or classical Chinese.'''
    global zhmodel
    if zhmodel is None:
        import json
        zhmodel = json.load(
            open(os.path.join(_curpath, 'modelzh.json'), 'r', encoding='utf-8'))
    cscore = 0
    mscore = 0
    for ch in s:
        ordch = ord(ch)
        if 0x4E00 <= ordch < 0x9FCD:
            cscore += zhmodel['zhc'][ordch - 0x4E00]
            mscore += zhmodel['zhm'][ordch - 0x4E00]
    return (cscore, mscore)


def checktxttype(cscore, mscore):
    if cscore > mscore:
        return 'c'
    elif cscore < mscore:
        return 'm'
    else:
        return None


def num2chinese(num, big=False, simp=True, o=False, twoalt=False):
    """
    Converts numbers to Chinese representations.

    `big`   : use financial characters.
    `simp`  : use simplified characters instead of traditional characters.
    `o`     : use 〇 for zero.
    `twoalt`: use 两/兩 for two when appropriate.

    Note that `o` and `twoalt` is ignored when `big` is used, 
    and `twoalt` is ignored when `o` is used for formal representations.
    """
    # check num first
    nd = str(num)
    if abs(float(nd)) >= 1e48:
        raise ValueError('number out of range')
    elif 'e' in nd:
        raise ValueError('scientific notation is not supported')
    c_symbol = '正负点' if simp else '正負點'
    if o:  # formal
        twoalt = False
    if big:
        c_basic = '零壹贰叁肆伍陆柒捌玖' if simp else '零壹貳參肆伍陸柒捌玖'
        c_unit1 = '拾佰仟'
        c_twoalt = '贰' if simp else '貳'
    else:
        c_basic = '〇一二三四五六七八九' if o else '零一二三四五六七八九'
        c_unit1 = '十百千'
        if twoalt:
            c_twoalt = '两' if simp else '兩'
        else:
            c_twoalt = '二'
    c_unit2 = '万亿兆京垓秭穰沟涧正载' if simp else '萬億兆京垓秭穰溝澗正載'
    revuniq = lambda l: ''.join(k for k, g in itertools.groupby(reversed(l)))
    nd = str(num)
    result = []
    if nd[0] == '+':
        result.append(c_symbol[0])
    elif nd[0] == '-':
        result.append(c_symbol[1])
    if '.' in nd:
        integer, remainder = nd.lstrip('+-').split('.')
    else:
        integer, remainder = nd.lstrip('+-'), None
    if int(integer):
        splitted = [integer[max(i - 4, 0):i]
                    for i in range(len(integer), 0, -4)]
        intresult = []
        for nu, unit in enumerate(splitted):
            # special cases
            if int(unit) == 0:  # 0000
                intresult.append(c_basic[0])
                continue
            elif nu > 0 and int(unit) == 2:  # 0002
                intresult.append(c_twoalt + c_unit2[nu - 1])
                continue
            ulist = []
            unit = unit.zfill(4)
            for nc, ch in enumerate(reversed(unit)):
                if ch == '0':
                    if ulist:  # ???0
                        ulist.append(c_basic[0])
                elif nc == 0:
                    ulist.append(c_basic[int(ch)])
                elif nc == 1 and ch == '1' and unit[1] == '0':
                    # special case for tens
                    # edit the 'elif' if you don't like
                    # 十四, 三千零十四, 三千三百一十四
                    ulist.append(c_unit1[0])
                elif nc > 1 and ch == '2':
                    ulist.append(c_twoalt + c_unit1[nc - 1])
                else:
                    ulist.append(c_basic[int(ch)] + c_unit1[nc - 1])
            ustr = revuniq(ulist)
            if nu == 0:
                intresult.append(ustr)
            else:
                intresult.append(ustr + c_unit2[nu - 1])
        result.append(revuniq(intresult).strip(c_basic[0]))
    else:
        result.append(c_basic[0])
    if remainder:
        result.append(c_symbol[2])
        result.append(''.join(c_basic[int(ch)] for ch in remainder))
    return ''.join(result)


stripquotes = lambda s: s.lstrip('"‘“「『').rstrip('"’”」』')
fw2hw = lambda s: ''.join(
    (chr(ord(ch) - 0xFEE0) if ord(ch) in fullwidth else ch) for ch in s)
hw2fw = lambda s: ''.join(
    (chr(ord(ch) + 0xFEE0) if ch in halfwidth else ch) for ch in s)


def _test_fixsplit():
    test = """从高祖父到曾孙称为“九族”。这“九族”代表着长幼尊卑秩序和家族血统的承续关系。
《诗》、《书》、《易》、《礼》、《春秋》，再加上《乐》称“六经”，这是中国古代儒家的重要经典，应当仔细阅读。
这就是：宇宙间万事万物循环变化的道理的书籍。
《连山》、《归藏》、《周易》，是我国古代的三部书，这三部书合称“三易”，“三易”是用“卦”的形式来说明宇宙间万事万物循环变化的道理的书籍。
登楼而望，慨然而叹曰：“容容其山，旅旅其石，与地终也!吁嗟人乎!病之蚀气也，如水浸火。
吾闻老聃多寿，尝读其书曰：‘吾惟无身，是以无患。’盖欲窃之而未能也”齐宣王见孟子于雪宫。
“昔者齐景公问于晏子曰：‘吾欲观于转附、朝舞，遵海而南，放于琅邪。吾何修而可以比于先王观也？’
高祖说：“该怎样对付呢？”陈平说：“古代天子有巡察天下，召集诸侯。南方有云梦这个地方，陛下只管假装外出巡游云梦，在陈地召集诸侯。陈地在楚国的西边边境上，韩信听说天子因为爱好外出巡游，看形势必然没有什么大事，就会到国境外来拜见陛下。拜见，陛下趁机抓住他，这只是一个力士的事情而已。”“不知道。”高祖认为有道理。
。他们就是这样的。
""".strip().split('\n')
    for s in test:
        print(fixmissing(splitsentence(s)))

if __name__ == '__main__':
    import sys
    _test_fixsplit()
    print(' '.join(addwallzone('《连山》、《归藏》、《周易》，是我国古代的三部书，这三部书合称“三易”，“三易”是用“卦”的形式来说明(宇宙间万事万物循环变化的道理的书籍。')))
    # print(checktxttype(sys.stdin.read()))

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import itertools

'''
Various functions for processing Chinese text.
'''

halfwidth = frozenset('!(),:;?')
fullwidth = frozenset(itertools.chain(
    range(0xFF02, 0xFF07 + 1),
    (0xFF0A, 0xFF0B, 0xFF0E, 0xFF0F, 0xFF1C, 0xFF1D,
     0xFF1E, 0xFF3C, 0xFF3E, 0xFF3F, 0xFF40),
    range(0xFF10, 0xFF19 + 1),
    range(0xFF20, 0xFF3A + 1),
    range(0xFF41, 0xFF5A + 1)))
resentencesp = re.compile('([﹒﹔﹖﹗．；。！？]["’”」』]{0,2}|：(?=["‘“「『]{1,2}|$))')
refixmissing = re.compile(
    '(^[^"‘“「『’”」』，；。！？]+["’”」』]|^["‘“「『]?[^"‘“「『’”」』]+[，；。！？][^"‘“「『‘“「『]*["’”」』])(?!["‘“「『’”」』，；。！？])')

punctstr = (
    '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¢£¥·ˇˉ―‖‘’“”•′‵、。々'
    '〈〉《》「」『』【】〔〕〖〗〝〞︰︱︳︴︵︶︷︸︹︺︻︼︽︾︿﹀﹁﹂﹃﹄'
    '﹏﹐﹒﹔﹕﹖﹗﹙﹚﹛﹜﹝﹞！（），．：；？［｛｜｝～､￠￡￥')

punct = frozenset(punctstr)

whitespace = ' \t\n\r\x0b\x0c\u3000'

resplitpunct = re.compile('([%s])' % re.escape(punctstr))

tailpunct = ('''!),-.:;?]}¢·ˇˉ―‖’”•′■□△○●'''
             '''、。々〉》」』】〕〗〞︰︱︳︴︶︸︺︼︾﹀﹂﹄﹏'''
             '''﹐﹒﹔﹕﹖﹗﹚﹜﹞！），．：；？｜］｝～､￠''') + whitespace
headpunct = ('''([`{£¥‘“〈《「『【〔〖〝'''
             '''︵︷︹︻︽︿﹁﹃﹙﹛﹝（［｛￡￥''') + whitespace

openbrckt = ('([{（［｛⦅〚⦃“‘‹«「〈《【〔⦗『〖〘｢⟦⟨⟪⟮⟬⌈⌊⦇⦉❛❝❨❪❴❬❮❰❲'
             '⏜⎴⏞〝︵⏠﹁﹃︹︻︗︿︽﹇︷〈⦑⧼﹙﹛﹝⁽₍⦋⦍⦏⁅⸢⸤⟅⦓⦕⸦⸨｟⧘⧚⸜⸌⸂⸄⸉᚛༺༼')
clozbrckt = (')]}）］｝⦆〛⦄”’›»」〉》】〕⦘』〗〙｣⟧⟩⟫⟯⟭⌉⌋⦈⦊❜❞❩❫❵❭❯❱❳'
             '⏝⎵⏟〞︶⏡﹂﹄︺︼︘﹀︾﹈︸〉⦒⧽﹚﹜﹞⁾₎⦌⦎⦐⁆⸣⸥⟆⦔⦖⸧⸩｠⧙⧛⸝⸍⸃⸅⸊᚜༻༽')

ucjk = frozenset(itertools.chain(
    range(0x1100, 0x11FF + 1),
    range(0x2E80, 0xA4CF + 1),
    range(0xA840, 0xA87F + 1),
    range(0xAC00, 0xD7AF + 1),
    range(0xF900, 0xFAFF + 1),
    range(0xFE30, 0xFE4F + 1),
    range(0xFF65, 0xFFDC + 1),
    range(0xFF01, 0xFF0F + 1),
    range(0xFF1A, 0xFF20 + 1),
    range(0xFF3B, 0xFF40 + 1),
    range(0xFF5B, 0xFF60 + 1),
    range(0x20000, 0x2FFFF + 1)
))

zhmodel = None
_curpath = os.path.normpath(
    os.path.join(os.getcwd(), os.path.dirname(__file__)))

RE_WS_IN_FW = re.compile(
    r'([‘’“”…─\u2e80-\u312f\u3200-\u32ff\u3400-\u4dbf\u4e00-\u9fff'
    '\uf900-\ufaff\ufe30-\ufe57\uff00-\uffef\U00020000-\U0002A6DF'
    '\U0002A700-\U0002B73F\U0002B740-\U0002B81F\U0002B820-\U0002CEAF'
    '\U0002F800-\U0002FA1F])\s+(?=[‘’“”…\u2e80-\u312f\u3200-\u32ff'
    '\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\ufe30-\ufe57\uff00-\uffef'
    '\U00020000-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F'
    '\U0002B820-\U0002CEAF\U0002F800-\U0002FA1F])')

RE_FW = re.compile(
    '([\u2018\u2019\u201c\u201d\u2026\u2e80-\u312f\u3200-\u32ff'
    '\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\ufe30-\ufe57\uff00-\uffef'
    '\U00020000-\U0002A6DF\U0002A700-\U0002B73F'
    '\U0002B740-\U0002B81F\U0002B820-\U0002CEAF'
    '\U0002F800-\U0002FA1F]+)')

RE_UCJK = re.compile(
    '([\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff'
    '\U00020000-\U0002A6DF\U0002A700-\U0002B73F'
    '\U0002B740-\U0002B81F\U0002B820-\U0002CEAF'
    '\U0002F800-\U0002FA1F]+)'
)

RE_CTRL = re.compile("[\000-\037\ufeff]+")

hasucjk = lambda s: RE_UCJK.search(s)
removectrl = lambda s: RE_CTRL.sub('', s)

detokenize = lambda s: RE_WS_IN_FW.sub(r'\1', s).strip()
detokenize.__doc__ = 'Detokenization function for Chinese.'

def splitsentence(sentence):
    '''Split a piece of Chinese into sentences.'''
    # s = ''.join((chr(ord(ch)+0xFEE0) if ch in halfwidth else ch) for ch in sentence)
    s = sentence
    slist = []
    for i in resentencesp.split(s):
        if resentencesp.match(i) and slist:
            slist[-1] += i
        elif i:
            slist.append(i)
    return slist


def splithard(sentence, maxchar=None):
    '''Forcely split a piece of Chinese into sentences with the limit of max sentence length.'''
    slist = splitsentence(sentence)
    if maxchar is None:
        return slist
    slist1 = []
    for sent in slist:
        if len(sent) > maxchar:
            for i in resplitpunct.split(sent):
                if resplitpunct.match(i) and slist1:
                    slist1[-1] += i
                elif i:
                    slist1.append(i)
        else:
            slist1.append(sent)
    slist = slist1
    slist1 = []
    for sent in slist:
        if len(sent) > maxchar:
            slist1.extend(sent[i:i + maxchar]
                          for i in range(0, len(sent), maxchar))
        else:
            slist1.append(sent)
    slist = slist1
    return slist


def fixmissing(slist):
    '''Fix missing quotes.'''
    newlist = []
    for i in slist:
        newlist.extend(filter(None, refixmissing.split(i)))
    return newlist


def filterlist(slist):
    '''Get meaningful sentences.'''
    for i in slist:
        s = i.lstrip(tailpunct).rstrip(headpunct)
        if len(s) > 1:
            yield s


def addwalls(tokiter):
    '''Add walls between punctuations for Moses.'''
    lastwall = False
    for tok in tokiter:
        if tok in punct:
            if not lastwall:
                yield '<wall />'
            yield tok
            yield '<wall />'
            lastwall = True
        else:
            yield tok
            lastwall = False


def addwallzone(tokiter):
    '''Add walls and zones between punctuations for Moses.'''
    W = '<wall />'
    out = []
    expect = zidx = None
    for tok in tokiter:
        if tok in punct:
            if not (out and out[-1] == W):
                out.append(W)
            if tok == expect:
                out[zidx] = '<zone>'
                out.append(tok)
                out.append('</zone>')
                expect = zidx = None
            else:
                bid = openbrckt.find(tok)
                if bid > -1:
                    expect = clozbrckt[bid]
                    zidx = len(out) - 1
                out.append(tok)
                out.append(W)
        else:
            out.append(tok)
    if out and out[0] == W:
        out.pop(0)
    if out and out[-1] == W:
        out.pop()
    return out


def calctxtstat(s):
    '''Detect whether a string is modern or classical Chinese.'''
    global zhmodel
    if zhmodel is None:
        import json
        zhmodel = json.load(
            open(os.path.join(_curpath, 'modelzh.json'), 'r', encoding='utf-8'))
    cscore = 0
    mscore = 0
    for ch in s:
        ordch = ord(ch)
        if 0x4E00 <= ordch < 0x9FCD:
            cscore += zhmodel['zhc'][ordch - 0x4E00]
            mscore += zhmodel['zhm'][ordch - 0x4E00]
    return (cscore, mscore)


def checktxttype(cscore, mscore):
    if cscore > mscore:
        return 'c'
    elif cscore < mscore:
        return 'm'
    else:
        return None


def num2chinese(num, big=False, simp=True, o=False, twoalt=False):
    """
    Converts numbers to Chinese representations.

    `big`   : use financial characters.
    `simp`  : use simplified characters instead of traditional characters.
    `o`     : use 〇 for zero.
    `twoalt`: use 两/兩 for two when appropriate.

    Note that `o` and `twoalt` is ignored when `big` is used, 
    and `twoalt` is ignored when `o` is used for formal representations.
    """
    # check num first
    nd = str(num)
    if abs(float(nd)) >= 1e48:
        raise ValueError('number out of range')
    elif 'e' in nd:
        raise ValueError('scientific notation is not supported')
    c_symbol = '正负点' if simp else '正負點'
    if o:  # formal
        twoalt = False
    if big:
        c_basic = '零壹贰叁肆伍陆柒捌玖' if simp else '零壹貳參肆伍陸柒捌玖'
        c_unit1 = '拾佰仟'
        c_twoalt = '贰' if simp else '貳'
    else:
        c_basic = '〇一二三四五六七八九' if o else '零一二三四五六七八九'
        c_unit1 = '十百千'
        if twoalt:
            c_twoalt = '两' if simp else '兩'
        else:
            c_twoalt = '二'
    c_unit2 = '万亿兆京垓秭穰沟涧正载' if simp else '萬億兆京垓秭穰溝澗正載'
    revuniq = lambda l: ''.join(k for k, g in itertools.groupby(reversed(l)))
    nd = str(num)
    result = []
    if nd[0] == '+':
        result.append(c_symbol[0])
    elif nd[0] == '-':
        result.append(c_symbol[1])
    if '.' in nd:
        integer, remainder = nd.lstrip('+-').split('.')
    else:
        integer, remainder = nd.lstrip('+-'), None
    if int(integer):
        splitted = [integer[max(i - 4, 0):i]
                    for i in range(len(integer), 0, -4)]
        intresult = []
        for nu, unit in enumerate(splitted):
            # special cases
            if int(unit) == 0:  # 0000
                intresult.append(c_basic[0])
                continue
            elif nu > 0 and int(unit) == 2:  # 0002
                intresult.append(c_twoalt + c_unit2[nu - 1])
                continue
            ulist = []
            unit = unit.zfill(4)
            for nc, ch in enumerate(reversed(unit)):
                if ch == '0':
                    if ulist:  # ???0
                        ulist.append(c_basic[0])
                elif nc == 0:
                    ulist.append(c_basic[int(ch)])
                elif nc == 1 and ch == '1' and unit[1] == '0':
                    # special case for tens
                    # edit the 'elif' if you don't like
                    # 十四, 三千零十四, 三千三百一十四
                    ulist.append(c_unit1[0])
                elif nc > 1 and ch == '2':
                    ulist.append(c_twoalt + c_unit1[nc - 1])
                else:
                    ulist.append(c_basic[int(ch)] + c_unit1[nc - 1])
            ustr = revuniq(ulist)
            if nu == 0:
                intresult.append(ustr)
            else:
                intresult.append(ustr + c_unit2[nu - 1])
        result.append(revuniq(intresult).strip(c_basic[0]))
    else:
        result.append(c_basic[0])
    if remainder:
        result.append(c_symbol[2])
        result.append(''.join(c_basic[int(ch)] for ch in remainder))
    return ''.join(result)


stripquotes = lambda s: s.lstrip('"‘“「『').rstrip('"’”」』')
fw2hw = lambda s: ''.join(
    (chr(ord(ch) - 0xFEE0) if ord(ch) in fullwidth else ch) for ch in s)
hw2fw = lambda s: ''.join(
    (chr(ord(ch) + 0xFEE0) if ch in halfwidth else ch) for ch in s)


def _test_fixsplit():
    test = """从高祖父到曾孙称为“九族”。这“九族”代表着长幼尊卑秩序和家族血统的承续关系。
《诗》、《书》、《易》、《礼》、《春秋》，再加上《乐》称“六经”，这是中国古代儒家的重要经典，应当仔细阅读。
这就是：宇宙间万事万物循环变化的道理的书籍。
《连山》、《归藏》、《周易》，是我国古代的三部书，这三部书合称“三易”，“三易”是用“卦”的形式来说明宇宙间万事万物循环变化的道理的书籍。
登楼而望，慨然而叹曰：“容容其山，旅旅其石，与地终也!吁嗟人乎!病之蚀气也，如水浸火。
吾闻老聃多寿，尝读其书曰：‘吾惟无身，是以无患。’盖欲窃之而未能也”齐宣王见孟子于雪宫。
“昔者齐景公问于晏子曰：‘吾欲观于转附、朝舞，遵海而南，放于琅邪。吾何修而可以比于先王观也？’
高祖说：“该怎样对付呢？”陈平说：“古代天子有巡察天下，召集诸侯。南方有云梦这个地方，陛下只管假装外出巡游云梦，在陈地召集诸侯。陈地在楚国的西边边境上，韩信听说天子因为爱好外出巡游，看形势必然没有什么大事，就会到国境外来拜见陛下。拜见，陛下趁机抓住他，这只是一个力士的事情而已。”“不知道。”高祖认为有道理。
。他们就是这样的。
""".strip().split('\n')
    for s in test:
        print(fixmissing(splitsentence(s)))

if __name__ == '__main__':
    import sys
    _test_fixsplit()
    print(' '.join(addwallzone('《连山》、《归藏》、《周易》，是我国古代的三部书，这三部书合称“三易”，“三易”是用“卦”的形式来说明(宇宙间万事万物循环变化的道理的书籍。')))
    # print(checktxttype(sys.stdin.read()))
#!/usr/bin/env python3 # -*- coding: utf-8 -*- Various functions for processing Chinese text. !),-.:;?]}¢·ˇˉ―‖’”•′■□△○● 、。々〉》」』】〕〗〞︰︱︳︴︶︸︺︼︾﹀﹂﹄﹏ ﹐﹒﹔﹕﹖﹗﹚﹜﹞！），．：；？｜］｝～､￠ ([`{£¥‘“〈《「『【〔〖〝 ︵︷︹︻︽︿﹁﹃﹙﹛﹝（［｛￡￥ Split a piece of Chinese into sentences. # s = ''.join((chr(ord(ch)+0xFEE0) if ch in halfwidth else ch) for ch in sentence) Forcely split a piece of Chinese into sentences with the limit of max sentence length. Fix missing quotes. Get meaningful sentences. Add walls between punctuations for Moses. Add walls and zones between punctuations for Moses. Detect whether a string is modern or classical Chinese. Converts numbers to Chinese representations.      `big`   : use financial characters.     `simp`  : use simplified characters instead of traditional characters.     `o`     : use 〇 for zero.     `twoalt`: use 两/兩 for two when appropriate.      Note that `o` and `twoalt` is ignored when `big` is used,      and `twoalt` is ignored when `o` is used for formal representations. # check num first # formal # special cases # 0000 # 0002 # ???0 # special case for tens # edit the 'elif' if you don't like # 十四, 三千零十四, 三千三百一十四 从高祖父到曾孙称为“九族”。这“九族”代表着长幼尊卑秩序和家族血统的承续关系。 《诗》、《书》、《易》、《礼》、《春秋》，再加上《乐》称“六经”，这是中国古代儒家的重要经典，应当仔细阅读。 这就是：宇宙间万事万物循环变化的道理的书籍。 《连山》、《归藏》、《周易》，是我国古代的三部书，这三部书合称“三易”，“三易”是用“卦”的形式来说明宇宙间万事万物循环变化的道理的书籍。 登楼而望，慨然而叹曰：“容容其山，旅旅其石，与地终也!吁嗟人乎!病之蚀气也，如水浸火。 吾闻老聃多寿，尝读其书曰：‘吾惟无身，是以无患。’盖欲窃之而未能也”齐宣王见孟子于雪宫。 “昔者齐景公问于晏子曰：‘吾欲观于转附、朝舞，遵海而南，放于琅邪。吾何修而可以比于先王观也？’ 高祖说：“该怎样对付呢？”陈平说：“古代天子有巡察天下，召集诸侯。南方有云梦这个地方，陛下只管假装外出巡游云梦，在陈地召集诸侯。陈地在楚国的西边边境上，韩信听说天子因为爱好外出巡游，看形势必然没有什么大事，就会到国境外来拜见陛下。拜见，陛下趁机抓住他，这只是一个力士的事情而已。”“不知道。”高祖认为有道理。 。他们就是这样的。 # print(checktxttype(sys.stdin.read()))