# https://github.com/polm/cutlet/blob/master/cutlet/cutlet.py
from dataclasses import dataclass
from fugashi import Tagger
from num2kana import Convert
import mojimoji
import re
import unicodedata
HEPBURN = {
chr(12449):'a', #ァ
chr(12450):'a', #ア
chr(12451):'i', #ィ
chr(12452):'i', #イ
chr(12453):'ɯ', #ゥ
chr(12454):'ɯ', #ウ
chr(12455):'e', #ェ
chr(12456):'e', #エ
chr(12457):'o', #ォ
chr(12458):'o', #オ
chr(12459):'ka', #カ
chr(12460):'ɡa', #ガ
chr(12461):'ki', #キ
chr(12462):'ɡi', #ギ
chr(12463):'kɯ', #ク
chr(12464):'ɡɯ', #グ
chr(12465):'ke', #ケ
chr(12466):'ɡe', #ゲ
chr(12467):'ko', #コ
chr(12468):'ɡo', #ゴ
chr(12469):'sa', #サ
chr(12470):'za', #ザ
chr(12471):'ɕi', #シ
chr(12472):'dʑi', #ジ
chr(12473):'sɨ', #ス
chr(12474):'zɨ', #ズ
chr(12475):'se', #セ
chr(12476):'ze', #ゼ
chr(12477):'so', #ソ
chr(12478):'zo', #ゾ
chr(12479):'ta', #タ
chr(12480):'da', #ダ
chr(12481):'tɕi', #チ
chr(12482):'dʑi', #ヂ
# chr(12483) #ッ
chr(12484):'tsɨ', #ツ
chr(12485):'zɨ', #ヅ
chr(12486):'te', #テ
chr(12487):'de', #デ
chr(12488):'to', #ト
chr(12489):'do', #ド
chr(12490):'na', #ナ
chr(12491):'ɲi', #ニ
chr(12492):'nɯ', #ヌ
chr(12493):'ne', #ネ
chr(12494):'no', #ノ
chr(12495):'ha', #ハ
chr(12496):'ba', #バ
chr(12497):'pa', #パ
chr(12498):'çi', #ヒ
chr(12499):'bi', #ビ
chr(12500):'pi', #ピ
chr(12501):'ɸɯ', #フ
chr(12502):'bɯ', #ブ
chr(12503):'pɯ', #プ
chr(12504):'he', #ヘ
chr(12505):'be', #ベ
chr(12506):'pe', #ペ
chr(12507):'ho', #ホ
chr(12508):'bo', #ボ
chr(12509):'po', #ポ
chr(12510):'ma', #マ
chr(12511):'mi', #ミ
chr(12512):'mɯ', #ム
chr(12513):'me', #メ
chr(12514):'mo', #モ
chr(12515):'ja', #ャ
chr(12516):'ja', #ヤ
chr(12517):'jɯ', #ュ
chr(12518):'jɯ', #ユ
chr(12519):'jo', #ョ
chr(12520):'jo', #ヨ
chr(12521):'ra', #ラ
chr(12522):'ri', #リ
chr(12523):'rɯ', #ル
chr(12524):'re', #レ
chr(12525):'ro', #ロ
chr(12526):'wa', #ヮ
chr(12527):'wa', #ワ
chr(12528):'i', #ヰ
chr(12529):'e', #ヱ
chr(12530):'o', #ヲ
# chr(12531) #ン
chr(12532):'vɯ', #ヴ
chr(12533):'ka', #ヵ
chr(12534):'ke', #ヶ
}
assert len(HEPBURN) == 84 and all(i in {12483, 12531} or chr(i) in HEPBURN for i in range(12449, 12535))
for k, v in list(HEPBURN.items()):
HEPBURN[chr(ord(k)-96)] = v
assert len(HEPBURN) == 84*2
HEPBURN.update({
chr(12535):'va', #ヷ
chr(12536):'vi', #ヸ
chr(12537):'ve', #ヹ
chr(12538):'vo', #ヺ
})
assert len(HEPBURN) == 84*2+4 and all(chr(i) in HEPBURN for i in range(12535, 12539))
HEPBURN.update({
chr(12784):'kɯ', #ㇰ
chr(12785):'ɕi', #ㇱ
chr(12786):'sɨ', #ㇲ
chr(12787):'to', #ㇳ
chr(12788):'nɯ', #ㇴ
chr(12789):'ha', #ㇵ
chr(12790):'çi', #ㇶ
chr(12791):'ɸɯ', #ㇷ
chr(12792):'he', #ㇸ
chr(12793):'ho', #ㇹ
chr(12794):'mɯ', #ㇺ
chr(12795):'ra', #ㇻ
chr(12796):'ri', #ㇼ
chr(12797):'rɯ', #ㇽ
chr(12798):'re', #ㇾ
chr(12799):'ro', #ㇿ
})
assert len(HEPBURN) == 84*2+4+16 and all(chr(i) in HEPBURN for i in range(12784, 12800))
HEPBURN.update({
chr(12452)+chr(12455):'je', #イェ
chr(12454)+chr(12451):'wi', #ウィ
chr(12454)+chr(12455):'we', #ウェ
chr(12454)+chr(12457):'wo', #ウォ
chr(12461)+chr(12455):'kʲe', #キェ
chr(12461)+chr(12515):'kʲa', #キャ
chr(12461)+chr(12517):'kʲɨ', #キュ
chr(12461)+chr(12519):'kʲo', #キョ
chr(12462)+chr(12515):'ɡʲa', #ギャ
chr(12462)+chr(12517):'ɡʲɨ', #ギュ
chr(12462)+chr(12519):'ɡʲo', #ギョ
chr(12463)+chr(12449):'kʷa', #クァ
chr(12463)+chr(12451):'kʷi', #クィ
chr(12463)+chr(12455):'kʷe', #クェ
chr(12463)+chr(12457):'kʷo', #クォ
chr(12464)+chr(12449):'ɡʷa', #グァ
chr(12464)+chr(12451):'ɡʷi', #グィ
chr(12464)+chr(12455):'ɡʷe', #グェ
chr(12464)+chr(12457):'ɡʷo', #グォ
chr(12471)+chr(12455):'ɕe', #シェ
chr(12471)+chr(12515):'ɕa', #シャ
chr(12471)+chr(12517):'ɕɨ', #シュ
chr(12471)+chr(12519):'ɕo', #ショ
chr(12472)+chr(12455):'dʑe', #ジェ
chr(12472)+chr(12515):'dʑa', #ジャ
chr(12472)+chr(12517):'dʑɨ', #ジュ
chr(12472)+chr(12519):'dʑo', #ジョ
chr(12481)+chr(12455):'tɕe', #チェ
chr(12481)+chr(12515):'tɕa', #チャ
chr(12481)+chr(12517):'tɕɨ', #チュ
chr(12481)+chr(12519):'tɕo', #チョ
chr(12482)+chr(12515):'dʑa', #ヂャ
chr(12482)+chr(12517):'dʑɨ', #ヂュ
chr(12482)+chr(12519):'dʑo', #ヂョ
chr(12484)+chr(12449):'tsa', #ツァ
chr(12484)+chr(12451):'tsi', #ツィ
chr(12484)+chr(12455):'tse', #ツェ
chr(12484)+chr(12457):'tso', #ツォ
chr(12486)+chr(12451):'ti', #ティ
chr(12486)+chr(12517):'tʲɨ', #テュ
chr(12487)+chr(12451):'di', #ディ
chr(12487)+chr(12517):'dʲɨ', #デュ
chr(12488)+chr(12453):'tɯ', #トゥ
chr(12489)+chr(12453):'dɯ', #ドゥ
chr(12491)+chr(12455):'ɲe', #ニェ
chr(12491)+chr(12515):'ɲa', #ニャ
chr(12491)+chr(12517):'ɲɨ', #ニュ
chr(12491)+chr(12519):'ɲo', #ニョ
chr(12498)+chr(12455):'çe', #ヒェ
chr(12498)+chr(12515):'ça', #ヒャ
chr(12498)+chr(12517):'çɨ', #ヒュ
chr(12498)+chr(12519):'ço', #ヒョ
chr(12499)+chr(12515):'bʲa', #ビャ
chr(12499)+chr(12517):'bʲɨ', #ビュ
chr(12499)+chr(12519):'bʲo', #ビョ
chr(12500)+chr(12515):'pʲa', #ピャ
chr(12500)+chr(12517):'pʲɨ', #ピュ
chr(12500)+chr(12519):'pʲo', #ピョ
chr(12501)+chr(12449):'ɸa', #ファ
chr(12501)+chr(12451):'ɸi', #フィ
chr(12501)+chr(12455):'ɸe', #フェ
chr(12501)+chr(12457):'ɸo', #フォ
chr(12501)+chr(12517):'ɸʲɨ', #フュ
chr(12501)+chr(12519):'ɸʲo', #フョ
chr(12511)+chr(12515):'mʲa', #ミャ
chr(12511)+chr(12517):'mʲɨ', #ミュ
chr(12511)+chr(12519):'mʲo', #ミョ
chr(12522)+chr(12515):'rʲa', #リャ
chr(12522)+chr(12517):'rʲɨ', #リュ
chr(12522)+chr(12519):'rʲo', #リョ
chr(12532)+chr(12449):'va', #ヴァ
chr(12532)+chr(12451):'vi', #ヴィ
chr(12532)+chr(12455):'ve', #ヴェ
chr(12532)+chr(12457):'vo', #ヴォ
chr(12532)+chr(12517):'vʲɨ', #ヴュ
chr(12532)+chr(12519):'vʲo', #ヴョ
})
assert len(HEPBURN) == 84*2+4+16+76
for k, v in list(HEPBURN.items()):
if len(k) != 2:
continue
a, b = k
assert a in HEPBURN and b in HEPBURN, (a, b)
a = chr(ord(a)-96)
b = chr(ord(b)-96)
assert a in HEPBURN and b in HEPBURN, (a, b)
HEPBURN[a+b] = v
assert len(HEPBURN) == 84*2+4+16+76*2
HEPBURN.update({
# symbols
# 'ー': '-', # 長音符, only used when repeated
'。': '.',
'、': ',',
'？': '?',
'！': '!',
'「': '"',
'」': '"',
'『': '"',
'』': '"',
'：': ':',
'（': '(',
'）': ')',
'《': '(',
'》': ')',
'【': '[',
'】': ']',
'・': ' ',#'/',
'，': ',',
'～': '—',
'〜': '—',
'—': '—',
'«': '«',
'»': '»',
# other
'゚': '', # combining handakuten by itself, just discard
'゙': '', # combining dakuten by itself
})
def add_dakuten(kk):
"""Given a kana (single-character string), add a dakuten."""
try:
# ii = 'かきくけこさしすせそたちつてとはひふへほ'.index(kk)
ii = 'カキクケコサシスセソタチツテトハヒフヘホ'.index(kk)
return 'ガギグゲゴザジズゼゾダヂヅデドバビブベボ'[ii]
# return 'がぎぐげござじずぜぞだぢづでどばびぶべぼ'[ii]
except ValueError:
# this is normal if the input is nonsense
return None
SUTEGANA = 'ャュョァィゥェォ' #'ゃゅょぁぃぅぇぉ'
PUNCT = '\'".!?(),;:-'
ODORI = '々〃ゝゞヽゞ'
@dataclass
class Token:
surface: str
space: bool # if a space should follow
def __str__(self):
sp = " " if self.space else ""
return f"{self.surface}{sp}"
class Katsu:
def __init__(self):
"""Create a Katsu object, which holds configuration as well as
tokenizer state.
Typical usage:
```python
katsu = Katsu()
roma = katsu.romaji("カツカレーを食べた")
# "Cutlet curry wo tabeta"
```
"""
self.tagger = Tagger()
self.table = dict(HEPBURN) # make a copy so we can modify it
self.exceptions = {}
def romaji(self, text):
"""Build a complete string from input text."""
if not text:
return ''
text = self._normalize_text(text)
words = self.tagger(text)
tokens = self._romaji_tokens(words)
out = ''.join([str(tok) for tok in tokens])
return re.sub(r'\s+', ' ', out.strip())
def phonemize(self, texts):
# espeak-ng API
return [self.romaji(text) for text in texts]
def _normalize_text(self, text):
"""Given text, normalize variations in Japanese.
This specifically removes variations that are meaningless for romaji
conversion using the following steps:
- Unicode NFKC normalization
- Full-width Latin to half-width
- Half-width katakana to full-width
"""
# perform unicode normalization
text = re.sub(r'[〜～](?=\d)', 'から', text) # wave dash range
text = unicodedata.normalize('NFKC', text)
# convert all full-width alphanum to half-width, since it can go out as-is
text = mojimoji.zen_to_han(text, kana=False)
# replace half-width katakana with full-width
text = mojimoji.han_to_zen(text, digit=False, ascii=False)
return ''.join([(' '+Convert(t)) if t.isdigit() else t for t in re.findall(r'\d+|\D+', text)])
def _romaji_tokens(self, words):
"""Build a list of tokens from input nodes."""
out = []
for wi, word in enumerate(words):
po = out[-1] if out else None
pw = words[wi - 1] if wi > 0 else None
nw = words[wi + 1] if wi < len(words) - 1 else None
roma = self._romaji_word(word)
tok = Token(roma, False)
# handle punctuation with atypical spacing
surface = word.surface#['orig']
if surface in '「『' or roma in '([':
if po:
po.space = True
elif surface in '」』' or roma in ']).,?!:':
if po:
po.space = False
tok.space = True
elif roma == ' ':
tok.space = False
else:
tok.space = True
out.append(tok)
# remove any leftover sokuon
for tok in out:
tok.surface = tok.surface.replace(chr(12483), '')
return out
def _romaji_word(self, word):
"""Return the romaji for a single word (node)."""
surface = word.surface#['orig']
if surface in self.exceptions:
return self.exceptions[surface]
assert not surface.isdigit(), surface
if surface.isascii():
return surface
kana = word.feature.pron or word.feature.kana or surface
if word.is_unk:
if word.char_type == 7: # katakana
pass
elif word.char_type == 3: # symbol
return ''.join(map(lambda c: self.table.get(c, c), surface))
else:
return '' # TODO: silently fail
out = ''
for ki, char in enumerate(kana):
nk = kana[ki + 1] if ki < len(kana) - 1 else None
pk = kana[ki - 1] if ki > 0 else None
out += self._get_single_mapping(pk, char, nk)
return out
def _get_single_mapping(self, pk, kk, nk):
"""Given a single kana and its neighbors, return the mapped romaji."""
# handle odoriji
# NOTE: This is very rarely useful at present because odoriji are not
# left in readings for dictionary words, and we can't follow kana
# across word boundaries.
if kk in ODORI:
if kk in 'ゝヽ':
if pk: return pk
else: return '' # invalid but be nice
if kk in 'ゞヾ': # repeat with voicing
if not pk: return ''
vv = add_dakuten(pk)
if vv: return self.table[vv]
else: return ''
# remaining are 々 for kanji and 〃 for symbols, but we can't
# infer their span reliably (or handle rendaku)
return ''
# handle digraphs
if pk and (pk + kk) in self.table:
return self.table[pk + kk]
if nk and (kk + nk) in self.table:
return ''
if nk and nk in SUTEGANA:
if kk == 'ッ': return '' # never valid, just ignore
return self.table[kk][:-1] + self.table[nk]
if kk in SUTEGANA:
return ''
if kk == 'ー': # 長音符
return 'ː'
if ord(kk) in {12387, 12483}: # っ or ッ
tnk = self.table.get(nk)
if tnk and tnk[0] in 'bdɸɡhçijkmnɲoprstɯvwz':
return tnk[0]
return kk
if ord(kk) in {12435, 12531}: # ん or ン
# https://en.wikipedia.org/wiki/N_(kana)
# m before m,p,b
# ŋ before k,g
# ɲ before ɲ,tɕ,dʑ
# n before n,t,d,r,z
# ɴ otherwise
tnk = self.table.get(nk)
if tnk:
if tnk[0] in 'mpb':
return 'm'
elif tnk[0] in 'kɡ':
return 'ŋ'
elif any(tnk.startswith(p) for p in ('ɲ','tɕ','dʑ')):
return 'ɲ'
elif tnk[0] in 'ntdrz':
return 'n'
return 'ɴ'
return self.table.get(kk, '')