Upload 2 files

translation_pipeline.py

@@ -0,0 +1,159 @@
+from transformers import TranslationPipeline as HfTranslationPipeline
+from transformers.pipelines.text2text_generation import ReturnType
+import re
+
+
+hans_chars = set(
+    "万与专业丛东丝丢两严丧个丬丰临为丽举么义乌乐乔习乡书买乱争于亏亘亚产亩亲亵亸亿仅从仑仓仪们价众优会伛伞伟传伤伥伦伧伪伫体佥侠侣侥侦侧侨侩侪侬俣俦俨俩俪俭债倾偬偻偾偿傥傧储傩儿兑兖党兰关兴兹养兽冁内冈册写军农冢冯冲决况冻净凄凉减凑凛凤凫凭凯击凼凿刍刘则刚创删别刬刭刽刿剀剂剐剑剥剧劝办务劢动励劲劳势勋勐勚匀匦匮区医华协单卖卢卤卫却卺厂厅历厉压厌厍厕厢厣厦厨厩厮县参叆叇双发变叙叠叶号叹叽吁吕吗吣吨听启吴呒呓呕呖呗员呙呛呜咏咙咛咝咤咴哌哑哒哓哔哕哗哙哜哝哟唛唝唠唡唢唣唤唿啧啬啭啮啰啴啸喷喽喾嗫嗳嘘嘤嘱噜嚣嚯团园囱围囵国图圆圣圹场坂坏块坚坛坜坝坞坟坠垄垅垆垒垦垧垩垫垭垯垱垲垴埘埙埚埝埯堑堕塆墙壮声壳壶壸处备复够头夸夹夺奁奂奋奖奥妆妇妈妩妪妫姗娄娅娆娇娈娱娲娴婳婴婵婶媪嫒嫔嫱嬷孙学孪宁宝实宠审宪宫宽宾寝对寻导寿将尔尘尧尴尸尽层屃屉届属屡屦屿岁岂岖岗岘岙岚岛岭岽岿峃峄峡峣峤峥峦崂崃崄崭嵘嵚嵛嵝嵴巅巩巯币帅师帏帐帘帜带帧帮帱帻帼幂幞并广庄庆庐庑库应庙庞废庼廪开异弃张弥弪弯弹强归当录彟彦彻径徕忆忏忧忾怀态怂怃怄怅怆怜总怼怿恋恳恶恸恹恺恻恼恽悦悫悬悭悯惊惧惨惩惫惬惭惮惯愍愠愤愦愿慑慭憷懑懒懔戆戋戏戗战戬户扦执扩扪扫扬扰抚抛抟抠抡抢护报担拟拢拣拥拦拧拨择挂挚挛挜挝挞挟挠挡挢挣挤挥挦捞损捡换捣据掳掴掷掸掺掼揽揿搀搁搂搅携摄摅摆摇摈摊撄撵撷撸撺擞攒敌敛数斋斓斩断无旧时旷旸昙昼昽显晋晒晓晔晕晖暂暧术机杀杂权条来杨杩极构枞枢枣枥枧枨枪枫枭柜柠柽栀栅标栈栉栊栋栌栎栏树栖样栾桊桠桡桢档桤桥桦桧桨桩梦梼梾检棂椁椟椠椤椭楼榄榇榈榉槚槛槟槠横樯樱橥橱橹橼檩欢欤欧歼殁殇残殒殓殚殡殴毁毂毕毙毡毵氇气氢氩氲汇汉汤汹沓沟没沣沤沥沦沧沨沩沪沵泞泪泶泷泸泺泻泼泽泾洁洒洼浃浅浆浇浈浉浊测浍济浏浐浑浒浓浔浕涛涝涞涟涠涡涢涣涤润涧涨涩淀渊渌渍渎渐渑渔渖渗温湾湿溃溅溆溇滗滚滞滟滠满滢滤滥滦滨滩滪漤潆潇潋潍潜潴澜濑濒灏灭灯灵灾灿炀炉炖炜炝点炼炽烁烂烃烛烟烦烧烨烩烫烬热焕焖焘煅煳熘爱爷牍牦牵牺犊犟状犷犸犹狈狍狝狞独狭狮狯狰狱狲猃猎猕猡猪猫猬献獭玑玙玚玛玮环现玱玺珉珏珐珑珰珲琎琏琐琼瑶瑷璎瓒瓮瓯电画畅畲畴疖疗疟疠疡疬疮疯疴痈痉痒痖痨痪痫瘅瘆瘗瘘瘪瘫瘾瘿癞癣癫癯皑皱皲盏盐监盖盗盘眍眦眬睁睐睑瞒瞩矫矶矾矿砀码砖砗砚砜砺砻砾础硁硕硖硗硙硚确硷碍碛碜碱碹磙礼祎祢祯祷祸禀禄禅离秃秆种积称秽秾稆税稣稳穑穷窃窍窑窜窝窥窦窭竖竞笃笋笔笕笺笼笾筑筚筛筜筝筹签简箓箦箧箨箩箪箫篑篓篮篱簖籁籴类籼粜粝粤粪粮糁糇紧絷纟纠纡红纣纤纥约级纨纩纪纫纬纭纮纯纰纱纲纳纴纵纶纷纸纹纺纻纼纽纾线绀绁绂练组绅细织终绉绊绋绌绍绎经绐绑绒结绔绕绖绗绘给绚绛络绝绞统绠绡绢绣绤绥绦继绨绩绪绫绬续绮绯绰绱绲绳维绵绶绷绸绹绺绻综绽绾绿缀缁缂缃缄缅缆缇缈缉缊缋缌缍缎缏缐缑缒缓缔缕编缗缘缙缚缛缜缝缞缟缠缡缢缣缤缥缦缧缨缩缪缫缬缭缮缯缰缱缲缳缴缵罂网罗罚罢罴羁羟羡翘翙翚耢耧耸耻聂聋职聍联聩聪肃肠肤肷肾肿胀胁胆胜胧胨胪胫胶脉脍脏脐脑脓脔脚脱脶脸腊腌腘腭腻腼腽腾膑臜舆舣舰舱舻艰艳艹艺节芈芗芜芦苁苇苈苋苌苍苎苏苘苹茎茏茑茔茕茧荆荐荙荚荛荜荞荟荠荡荣荤荥荦荧荨荩荪荫荬荭荮药莅莜莱莲莳莴莶获莸莹莺莼萚萝萤营萦萧萨葱蒇蒉蒋蒌蓝蓟蓠蓣蓥蓦蔷蔹蔺蔼蕲蕴薮藁藓虏虑虚虫虬虮虽虾虿蚀蚁蚂蚕蚝蚬蛊蛎蛏蛮蛰蛱蛲蛳蛴蜕蜗蜡蝇蝈蝉蝎蝼蝾螀螨蟏衅衔补衬衮袄袅袆袜袭袯装裆裈裢裣裤裥褛褴襁襕见观觃规觅视觇览觉觊觋觌觍觎觏觐觑觞触觯詟誉誊讠计订讣认讥讦讧讨让讪讫训议讯记讱讲讳讴讵讶讷许讹论讻讼讽设访诀证诂诃评诅识诇诈诉诊诋诌词诎诏诐译诒诓诔试诖诗诘诙诚诛诜话诞诟诠诡询诣诤该详诧诨诩诪诫诬语诮误诰诱诲诳说诵诶请诸诹诺读诼诽课诿谀谁谂调谄谅谆谇谈谊谋谌谍谎谏谐谑谒谓谔谕谖谗谘谙谚谛谜谝谞谟谠谡谢谣谤谥谦谧谨谩谪谫谬谭谮谯谰谱谲谳谴谵谶豮贝贞负贠贡财责贤败账货质贩贪贫贬购贮贯贰贱贲贳贴贵贶贷贸费贺贻贼贽贾贿赀赁赂赃资赅赆赇赈赉赊赋赌赍赎赏赐赑赒赓赔赕赖赗赘赙赚赛赜赝赞赟赠赡赢赣赪赵赶趋趱趸跃跄跖跞践跶跷跸跹跻踊踌踪踬踯蹑蹒蹰蹿躏躜躯车轧轨轩轪轫转轭轮软轰轱轲轳轴轵轶轷轸轹轺轻轼载轾轿辀辁辂较辄辅辆辇辈辉辊辋辌辍辎辏辐辑辒输辔辕辖辗辘辙辚辞辩辫边辽达迁过迈运还这进远违连迟迩迳迹适选逊递逦逻遗遥邓邝邬邮邹邺邻郄郏郐郑郓郦郧郸酝酦酱酽酾酿释鉴銮錾钆钇针钉钊钋钌钍钎钏钐钑钒钓钔钕钖钗钘钙钚钛钝钞钟钠钡钢钣钤钥钦钧钨钩钪钫钬钭钮钯钰钱钲钳钴钵钶钷钸钹钺钻钼钽钾钿铀铁铂铃铄铅铆铈铉铊铋铍铎铏铐铑铒铕铗铘铙铚铛铜铝铞铟铠铡铢铣铤铥铦铧铨铪铫铬铭铮铯铰铱铲铳铴铵银铷铸铹铺铻铼铽链铿销锁锂锃锄锅锆锇锈锉锊锋锌锍锎锏锐锑锒锓锔锕锖锗错锚锜锞锟锠锡锢锣锤锥锦锨锩锫锬锭键锯锰锱锲锳锴锵锶锷锸锹锺锻锼锽锾锿镀镁镂镃镆镇镈镉镊镌镍镎镏镐镑镒镕镖镗镙镚镛镜镝镞镟镠镡镢镣镤镥镦镧镨镩镪镫镬镭镮镯镰镱镲镳镴镶长门闩闪闫闬闭问闯闰闱闲闳间闵闶闷闸闹闺闻闼闽闾闿阀阁阂阃阄阅阆阇阈阉阊阋阌阍阎阏阐阑阒阓阔阕阖阗阘阙阚阛队阳阴阵阶际陆陇陈陉陕陧陨险随隐隶隽难雏雠雳雾霁霭靓静靥鞑鞒鞯鞴韦韧韨韩韪韫韬韵页顶顷顸项顺须顼顽顾顿颀颁颂颃预颅领颇颈颉颊颋颌颍颎颏颐频颒颓颔颕颖颗题颙颚颛颜额颞颟颠颡颢颣颤颥颦颧风飏飐飑飒飓飔飕飖飗飘飙飚飞飨餍饤饥饦饧饨饩饪饫饬饭饮饯饰饱饲饳饴饵饶饷饸饹饺饻饼饽饾饿馀馁馂馃馄馅馆馇馈馉馊馋馌馍馎馏馐馑馒馓馔馕马驭驮驯驰驱驲驳驴驵驶驷驸驹驺驻驼驽驾驿骀骁骂骃骄骅骆骇骈骉骊骋验骍骎骏骐骑骒骓骔骕骖骗骘骙骚骛骜骝骞骟骠骡骢骣骤骥骦骧髅髋髌鬓魇魉鱼鱽鱾鱿鲀鲁鲂鲄鲅鲆鲇鲈鲉鲊鲋鲌鲍鲎鲏鲐鲑鲒鲓鲔鲕鲖鲗鲘鲙鲚鲛鲜鲝鲞鲟鲠鲡鲢鲣鲤鲥鲦鲧鲨鲩鲪鲫鲬鲭鲮鲯鲰鲱鲲鲳鲴鲵鲶鲷鲸鲹鲺鲻鲼鲽鲾鲿鳀鳁鳂鳃鳄鳅鳆鳇鳈鳉鳊鳋鳌鳍鳎鳏鳐鳑鳒鳓鳔鳕鳖鳗鳘鳙鳛鳜鳝鳞鳟鳠鳡鳢鳣鸟鸠鸡鸢鸣鸤鸥鸦鸧鸨鸩鸪鸫鸬鸭鸮鸯鸰鸱鸲鸳鸴鸵鸶鸷鸸鸹鸺鸻鸼鸽鸾鸿鹀鹁鹂鹃鹄鹅鹆鹇鹈鹉鹊鹋鹌鹍鹎鹏鹐鹑鹒鹓鹔鹕鹖鹗鹘鹚鹛鹜鹝鹞鹟鹠鹡鹢鹣鹤鹥鹦鹧鹨鹩鹪鹫鹬鹭鹯鹰鹱鹲鹳鹴鹾麦麸黄黉黡黩黪黾鼋鼌鼍鼗鼹齄齐齑齿龀龁龂龃龄龅龆龇龈龉龊龋龌龙龚龛龟咨尝“”"
+)
+
+
+def fix_chinese_text_generation_space(text):
+    output_text = text
+    output_text = re.sub(
+        r'([\u3401-\u9FFF+——！，。？、~@#￥%…&*（）：；《）《》“”()»〔〕\-!$^*()_+|~=`{}\[\]:";\'<>?,.·\/\\])\s([^0-9a-zA-Z])',
+        r"\1\2",
+        output_text,
+    )
+    output_text = re.sub(
+        r'([^0-9a-zA-Z])\s([\u3401-\u9FFF+——！，。？、~@#￥%…&*（）：；《）《》“”()»〔〕\-!$^*()_+|~=`{}\[\]:";\'<>?,.·\/\\])',
+        r"\1\2",
+        output_text,
+    )
+    output_text = re.sub(
+        r'([\u3401-\u9FFF+——！，。？、~@#￥%…&*（）：；《）《》“”()»〔〕\-!$^*()_+|~=`{}\[\]:";\'<>?,.·\/\\])\s([a-zA-Z0-9])',
+        r"\1\2",
+        output_text,
+    )
+    output_text = re.sub(
+        r'([a-zA-Z0-9])\s([\u3401-\u9FFF+——！，。？、~@#￥%…&*（）：；《）《》“”()»〔〕\-!$^*()_+|~=`{}\[\]:";\'<>?,.·\/\\])',
+        r"\1\2",
+        output_text,
+    )
+    output_text = re.sub(r"＄\s([0-9])", r"＄\1", output_text)
+    output_text = re.sub(",", "，", output_text)
+    output_text = re.sub(
+        r"([0-9])，([0-9])", r"\1,\2", output_text
+    )  # fix comma in numbers
+    # fix multiple commas
+    output_text = re.sub(r"\s?[，]+\s?", "，", output_text)
+    output_text = re.sub(r"\s?[、]+\s?", "、", output_text)
+    # fix period
+    output_text = re.sub(r"\s?[。]+\s?", "。", output_text)
+    # fix ...
+    output_text = re.sub(r"\s?\.{3,}\s?", "...", output_text)
+    # fix exclamation mark
+    output_text = re.sub(r"\s?[!！]+\s?", "！", output_text)
+    # fix question mark
+    output_text = re.sub(r"\s?[?？]+\s?", "？", output_text)
+    # fix colon
+    output_text = re.sub(r"\s?[:：]+\s?", "：", output_text)
+    # fix quotation mark
+    output_text = re.sub(r'\s?(["“”\']+)\s?', r"\1", output_text)
+    # fix semicolon
+    output_text = re.sub(r"\s?[;；]+\s?", "；", output_text)
+    # fix dots
+    output_text = re.sub(r"\s?([~●．…]+)\s?", r"\1", output_text)
+    output_text = re.sub(r"\s?\[…\]\s?", "", output_text)
+    output_text = re.sub(r"\s?\[\.\.\.\]\s?", "", output_text)
+    output_text = re.sub(r"\s?\.{3,}\s?", "...", output_text)
+    # fix slash
+    output_text = re.sub(r"\s?[／/]+\s?", "／", output_text)
+    # fix dollar sign
+    output_text = re.sub(r"\s?[$＄]+\s?", "＄", output_text)
+    # fix @
+    output_text = re.sub(r"\s?([@＠]+)\s?", "＠", output_text)
+    # fix baskets
+    output_text = re.sub(r"\s?([\[\(<��【「『（）』」】〗>\)\]]+)\s?", r"\1", output_text)
+
+    return output_text
+
+
+class TranslationPipeline(HfTranslationPipeline):
+    def __init__(
+        self,
+        model,
+        tokenizer,
+        device=None,
+        max_length=512,
+        src_lang=None,
+        tgt_lang=None,
+        num_beams=3,
+        do_sample=True,
+        top_k=50,
+        top_p=0.95,
+        temperature=1.0,
+        repetition_penalty=1.0,
+        length_penalty=1.0,
+        sequence_bias=None,
+        bad_words_ids=None,
+        no_repeat_ngram_size=0,
+    ):
+        self.model = model
+        self.tokenizer = tokenizer
+
+        def get_tokens(word):
+            return tokenizer([word], add_special_tokens=False).input_ids[0]
+
+        bad_words_ids = [get_tokens(char) for char in hans_chars]
+
+        super().__init__(
+            self.model,
+            self.tokenizer,
+            device=device,
+            max_length=max_length,
+            src_lang=src_lang,
+            tgt_lang=tgt_lang,
+            num_beams=num_beams,
+            do_sample=do_sample,
+            top_k=top_k if do_sample == True else None,
+            top_p=top_p if do_sample == True else None,
+            temperature=temperature,
+            repetition_penalty=repetition_penalty,
+            length_penalty=length_penalty,
+            sequence_bias=sequence_bias,
+            bad_words_ids=bad_words_ids,
+            no_repeat_ngram_size=no_repeat_ngram_size,
+        )
+
+    def postprocess(
+        self,
+        model_outputs,
+        return_type=ReturnType.TEXT,
+        clean_up_tokenization_spaces=True,
+    ):
+        records = super().postprocess(
+            model_outputs,
+            return_type=return_type,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+        )
+        for rec in records:
+            translation_text = fix_chinese_text_generation_space(
+                rec["translation_text"].strip()
+            )
+
+            rec["translation_text"] = translation_text
+        return records
+
+    def __call__(self, *args, **kwargs):
+        records = super().__call__(*args, **kwargs)
+
+        return records
+
+
+if __name__ == "__main__":
+    from transformers import BertTokenizerFast
+    from optimum.onnxruntime import ORTModelForSeq2SeqLM
+
+    model_id = "hon9kon9ize/bart-translation-zh-yue-onnx"
+
+    tokenizer = BertTokenizerFast.from_pretrained(model_id)
+    model = ORTModelForSeq2SeqLM.from_pretrained(model_id, use_cache=False)
+    pipe = TranslationPipeline(model=model, tokenizer=tokenizer)
+
+    print(
+        pipe(
+            "近年成为许多港人热门移居地的英国中部城巿诺定咸（又译诺丁汉，Nottingham），多年来一直面对财政困境，市议会周三（11月29日）宣布破产，是继英国第二大城市伯明翰今年9月宣布破产后，近期「爆煲」的另一个英国主要城市。诺定咸除了维持法例规定必须提供的服务外，巿政府将暂停所有非必要的公共开支。",
+            max_length=300,
+        )
+    )

translator.py

@@ -0,0 +1,439 @@
+from collections.abc import Callable
+import traceback
+from typing import List, Union
+from datasets import Dataset
+import re
+import pickle
+import os
+from transformers.pipelines.pt_utils import KeyDataset
+from transformers import AutoTokenizer
+from tqdm.auto import tqdm
+
+URL_REGEX = r"\b(https?://\S+)\b"
+EMAIL_REGEX = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
+TAG_REGEX = r"<[^>]+>"
+HANDLE_REGEX = r"[^a-zA-Z](@\w+)"
+
+
+class Translator:
+    def __init__(
+        self,
+        pipe: Callable,
+        max_length: int = 500,
+        batch_size: int = 16,
+        save_every_step=100,
+        text_key="text",
+        save_filename=None,
+        replace_chinese_puncts=False,
+        verbose=False,
+    ):
+        self.pipe = pipe
+        self.max_length = max_length
+        self.batch_size = batch_size
+        self.save_every_step = save_every_step
+        self.save_filename = save_filename
+        self.text_key = text_key
+        self.replace_chinese_puncts = replace_chinese_puncts
+        self.verbose = verbose
+
+        if max_length == None and hasattr(pipe.model.config, "max_length"):
+            self.max_length = pipe.model.config.max_length
+
+    def _is_chinese(self, text: str) -> bool:
+        return (
+            re.search(
+                r"[\u4e00-\u9fff\u3400-\u4dbf\U00020000-\U0002a6df\U0002a700-\U0002ebef\U00030000-\U000323af\ufa0e\ufa0f\ufa11\ufa13\ufa14\ufa1f\ufa21\ufa23\ufa24\ufa27\ufa28\ufa29\u3006\u3007][\ufe00-\ufe0f\U000e0100-\U000e01ef]?",
+                text,
+            )
+            is not None
+        )
+
+    def _split_sentences(self, text: str) -> List[str]:
+        tokens = self.pipe.tokenizer(text, add_special_tokens=False)
+        token_size = len(tokens.input_ids)
+
+        if len(text) <= self.max_length:
+            return [text]
+
+        delimiter = set()
+        delimiter.update("。！？；…!?;")
+        sent_list = []
+        sent = text
+
+        while token_size > self.max_length:
+            orig_sent_len = token_size
+
+            # find the index of delimiter near the max_length
+            for i in range(token_size - 2, 0, -1):
+                token = tokens.token_to_chars(0, i)
+                char = sent[token.start : token.end]
+
+                if char in delimiter:
+                    split_char_index = token.end
+                    next_sent = sent[split_char_index:]
+
+                    if len(next_sent) == 1:
+                        continue
+
+                    sent_list = [next_sent] + sent_list
+                    sent = sent[0:split_char_index]
+                    break
+
+            tokens = self.pipe.tokenizer(sent, add_special_tokens=False)
+            token_size = len(tokens.input_ids)
+
+            # no delimiter found, leave the sentence as it is
+            if token_size == orig_sent_len:
+                sent_list = [sent] + sent_list
+                sent = ""
+                break
+
+        if len(sent) > 0:
+            sent_list = [sent] + sent_list
+
+        return sent_list
+
+    def _preprocess(self, text: str) -> (str, str):
+        # extract entities
+        tags = re.findall(TAG_REGEX, text)
+        handles = re.findall(HANDLE_REGEX, text)
+        urls = re.findall(URL_REGEX, text)
+        emails = re.findall(EMAIL_REGEX, text)
+        entities = urls + emails + tags + handles
+
+        # escaple placeholders
+        text = re.sub(r"$e\[(\d+)\]", r"$E[\1]", text)
+
+        for i, entity in enumerate(entities):
+            text = text.replace(entity, "$e[%d]" % i, 1)
+
+        lines = text.split("\n")
+        sentences = []
+        num_tokens = []
+        template = text.replace("{", "{{").replace("}", "}}")
+        chunk_index = 0
+
+        for line in lines:
+            sentence = line.strip()
+
+            if len(sentence) > 0 and self._is_chinese(sentence):
+                chunks = self._split_sentences(sentence)
+
+                for chunk in chunks:
+                    sentences.append(chunk)
+                    tokens = self.pipe.tokenizer(chunk, add_special_tokens=False)
+                    num_tokens.append(len(tokens.input_ids))
+                    chunk = chunk.replace("{", "{{").replace("}", "}}")
+                    template = template.replace(chunk, "{%d}" % chunk_index, 1)
+                    chunk_index += 1
+
+        return sentences, template, num_tokens, entities
+
+    def _postprocess(
+        self,
+        template: str,
+        src_sentences: List[str],
+        translations: List[str],
+        entities: List[str],
+    ) -> str:
+        processed = []
+        alphanumeric_regex = re.compile(
+            "([a-zA-Zａ-ｚＡ-Ｚ０-９\d+'\",，（\(）\)：:；;“”。·\.\？?\！!‘’$\[\]<>/]+)"
+        )
+
+        def hash_text(text: List[str]) -> str:
+            text = "|".join(text)
+            puncts_map = str.maketrans("，；：（）。？！“”‘’", ",;:().?!\"\"''")
+            text = text.translate(puncts_map)
+            return text.lower()
+
+        for i, p in enumerate(translations):
+            src_sentence = src_sentences[i]
+
+            if self.replace_chinese_puncts:
+                p = re.sub(",", "，", p)  # replace all commas
+                p = re.sub(";", "；", p)  # replace semi-colon
+                p = re.sub(":", "：", p)  # replace colon
+                p = re.sub("\(", "（", p)  # replace round basket
+                p = re.sub("\)", "）", p)  # replace round basket
+                p = re.sub(r"([\d])，([\d])", r"\1,\2", p)
+
+            src_matches = re.findall(alphanumeric_regex, src_sentence)
+            tgt_matches = re.findall(alphanumeric_regex, p)
+
+            # length not match or no match
+            if (
+                len(src_matches) != len(tgt_matches)
+                or len(src_matches) == 0
+                or len(tgt_matches) == 0
+            ):
+                processed.append(p)
+                continue
+
+            # normalize full-width to half-width and lower case
+            src_hashes = hash_text(src_matches)
+            translated_hashes = hash_text(tgt_matches)
+
+            if src_hashes != translated_hashes:
+                # fix unmatched
+                for j in range(len(src_matches)):
+                    if src_matches[j] != tgt_matches[j]:
+                        p = p.replace(tgt_matches[j], src_matches[j], 1)
+
+            processed.append(p)
+
+        output = template.format(*processed)
+
+        # replace entities
+        for i, entity in enumerate(entities):
+            output = output.replace("$e[%d]" % i, entity, 1)
+
+        # unescape entity placeholders
+        output = re.sub(r"$E\[(\d+)\]", r"$e[\1]", output)
+
+        # fix repeated punctuations
+        output = re.sub(r"([「」（）『』《》。，：])\1+", r"\1", output)
+
+        # fix brackets
+        if "“" in output:
+            output = re.sub("“", "「", output)
+        if "”" in output:
+            output = re.sub("”", "」", output)
+
+        return output
+
+    def _save(self, translations):
+        with open(self.save_filename, "wb") as f:
+            pickle.dump(translations, f)
+
+    def __call__(self, inputs: Union[List[str], Dataset]) -> List[str]:
+        templates = []
+        sentences = []
+        num_tokens = []
+        sentence_indices = []
+        outputs = []
+        translations = []
+        entities_list = []
+        resume_from_file = None
+
+        if isinstance(inputs, Dataset):
+            ds = inputs
+        else:
+            if isinstance(inputs, str):
+                inputs = [inputs]
+            ds = Dataset.from_list([{"text": text} for text in inputs])
+
+        for i, text_input in tqdm(
+            enumerate(ds), total=len(ds), desc="Preprocessing", disable=not self.verbose
+        ):
+            chunks, template, num_tokens, entities = self._preprocess(
+                text_input["text"]
+            )
+            templates.append(template)
+            sentence_indices.append([])
+            entities_list.append(entities)
+
+            for j, chunk in enumerate(chunks):
+                sentences.append(chunk)
+                sentence_indices[len(sentence_indices) - 1].append(len(sentences) - 1)
+                num_tokens.append(num_tokens[j])
+
+        if self.save_filename:
+            resume_from_file = (
+                self.save_filename if os.path.isfile(self.save_filename) else None
+            )
+
+        if resume_from_file != None:
+            translations = pickle.load(open(resume_from_file, "rb"))
+
+        if self.verbose:
+            print("translated:", len(translations))
+            print("to translate:", len(sentences) - len(translations))
+
+            if resume_from_file != None:
+                print(
+                    "Resuming from {}({} records)".format(
+                        resume_from_file, len(translations)
+                    )
+                )
+
+        ds = Dataset.from_list(
+            [{"text": text} for text in sentences[len(translations) :]]
+        )
+
+        max_token_length = max(num_tokens)
+
+        if self.verbose:
+            print("Max Length:", max_token_length)
+
+        total_records = len(ds)
+
+        if total_records > 0:
+            step = 0
+
+            with tqdm(
+                disable=not self.verbose, desc="Translating", total=total_records
+            ) as pbar:
+                for out in self.pipe(
+                    KeyDataset(ds, self.text_key),
+                    batch_size=self.batch_size,
+                    max_length=self.max_length,
+                ):
+                    translations.append(out[0])
+
+                    # export generate result every n steps
+                    if (
+                        step != 0
+                        and self.save_filename != None
+                        and step % self.save_every_step == 0
+                    ):
+                        self._save(translations)
+
+                    step += 1
+
+                    pbar.update(1)
+
+        if self.save_filename != None and total_records > 0:
+            self._save(translations)
+
+        for i, template in tqdm(
+            enumerate(templates),
+            total=len(templates),
+            desc="Postprocessing",
+            disable=not self.verbose,
+        ):
+            try:
+                src_sentences = [sentences[index] for index in sentence_indices[i]]
+                tgt_sentences = [
+                    translations[index]["translation_text"]
+                    for index in sentence_indices[i]
+                ]
+                output = self._postprocess(
+                    template, src_sentences, tgt_sentences, entities_list[i]
+                )
+                outputs.append(output)
+            except Exception as error:
+                print(error)
+                print(template)
+                traceback.print_exc()
+                # print(template, sentence_indices[i], len(translations))
+
+        return outputs
+
+
+class Object(object):
+    pass
+
+
+class FakePipe(object):
+    def __init__(self, max_length: int = 500):
+        self.model = Object()
+        self.model.config = Object()
+        self.model.config.max_length = max_length
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            "indiejoseph/bart-translation-zh-yue"
+        )
+
+    def __call__(self, text: List[str], batch_size: str, max_length: int):
+        for i in range(len(text)):
+            sentence = text[i]
+            # extract entities
+            tags = re.findall(TAG_REGEX, sentence)
+            handles = re.findall(HANDLE_REGEX, sentence)
+            urls = re.findall(URL_REGEX, sentence)
+            emails = re.findall(EMAIL_REGEX, sentence)
+            entities = urls + emails + tags + handles
+
+            for i, entity in enumerate(entities):
+                sentence = sentence.replace(entity, "$e[%d]" % i, 1)
+
+            if "１２３" in sentence:
+                yield [{"translation_text": sentence.replace("１２３", "123")}]
+                continue
+            if "abc" in sentence:
+                yield [{"translation_text": sentence.replace("abc", "ABC")}]
+                continue
+            if "Acetaminophen" in sentence:
+                yield [
+                    {
+                        "translation_text": sentence.replace(
+                            "Acetaminophen", "ACEtaminidien"
+                        )
+                    }
+                ]
+                continue
+            yield [{"translation_text": sentence}]
+
+
+if __name__ == "__main__":
+    fake_pipe = FakePipe(60)
+
+    translator = Translator(fake_pipe, max_length=60, batch_size=2, verbose=True)
+
+    text1 = "对于编写聊天机器人的脚本，你可以采用不同的方法，包括使用基于规则的系统、自然语言处理（NLP）技术和机器学习模型。下面是一个简单的例子，展示如何使用基于规则的方法来构建一个简单的聊天机器人："
+    text2 = """对于编写聊天机器人的脚本，你可以采用不同的方法，包括使用基于规则的系统、自然语言处理（NLP）技术和机器学习模型。下面是一个简单的例子，展示如何使用基于规则的方法来构建一个简单的聊天机器人：
+
+```
+# 设置用于匹配输入的关键字，并定义相应的回答数据字典。
+keywords = {'你好': '你好！很高兴见到你。',
+           '再见': '再见！有机会再聊。',
+           '你叫什么': '我是一个聊天机器人。',
+           '你是谁': '我是一个基于人工智能技术制作的聊天机器人。'}
+
+# 定义用于处理用户输入的函数。
+def chatbot(input_text):
+    # 遍历关键字数据字典，匹配用户的输入。
+    for key in keywords:
+        if key in input_text:
+            # 如果匹配到了关键字，返回相应的回答。
+            return keywords[key]
+    # 如果没有找到匹配的关键字，返回默认回答。
+    return "对不起，我不知道你在说什么。"
+
+# 运行聊天机器人。
+while True:
+    # 获取用户输入。
+    user_input = input('用户: ')
+    # 如果用户输入“再见”，退出程序。
+    if user_input == '再见':
+        break
+    # 处理用户输入，并打印回答。
+    print('机器人: ' + chatbot(user_input))
+```
+
+这是一个非常简单的例子。对于实用的聊天机器人，可能需要使用更复杂的 NLP 技术和机器学习模型，以更好地理解和回答用户的问题。"""
+    text3 = "布洛芬(Ibuprofen)同撲熱息痛(Acetaminophen)係兩種常見嘅非處方藥，用於緩解疼痛、發燒同關節痛。"
+    text4 = "１２３ “abc” def's http://www.google.com [email protected] @abc 網址：http://localhost/abc下載"
+    text5 = "新力公司董事長盛田昭夫、自民黨國會議員石原慎太郎等人撰寫嘅《日本可以說「不」》、《日本還要說「不」》、《日本堅決說「不」》三本書中話道：「無啦啦挑起戰爭嘅好戰日本人，製造南京大屠殺嘅殘暴嘅日本人，呢d就係人地對日本人���兩個誤解，都係‘敲打日本’嘅兩個根由，我地必須採取措施消除佢。」"
+    outputs = translator([text1, text2, text3, text4, text5])
+
+    # for i, line in enumerate(outputs[1].split("\n")):
+    #     input_text = text2.split("\n")[i]
+
+    #     if line != input_text:
+    #         print(line, text2.split("\n")[i])
+
+    assert outputs[0] == text1
+    assert outputs[1] == text2.replace("“", "「").replace("”", "」")
+    assert outputs[2] == text3
+    assert outputs[3] == text4.replace("“", "「").replace("”", "」")
+    assert outputs[4] == text5
+
+    # exception
+    assert (
+        len(
+            translator._split_sentences(
+                "新力公司董事長盛田昭夫、自民黨國會議員石原慎太郎等人撰寫嘅《日本可以說「不」》、《日本還要說「不」》、《日本堅決說「不」》三本書中話道：「無啦啦挑起戰爭嘅好戰日本人，製造南京大屠殺嘅殘暴嘅日本人，呢d就係人地對日本人嘅兩個誤解，都係‘敲打日本’嘅兩個根由，我地必須採取措施消除佢。」"
+            )
+        )
+        == 1
+    )
+
+    translator = Translator(fake_pipe, max_length=5, batch_size=2, verbose=True)
+
+    assert (
+        len(
+            translator._split_sentences("====。====。====。====。====。====。====。====。====。")
+        )
+        == 9
+    )