#! /usr/bin/python3
src="KoichiYasuoka/modernbert-base-thai-wikipedia-upos"
tgt="KoichiYasuoka/modernbert-base-thai-wikipedia-ud-triangular"
import os
os.system("""D=spaCy-Thai/UD_Thai-Corpora
test -d $D || git clone --depth=1 https://github.com/KoichiYasuoka/spaCy-Thai
nawk 'BEGIN{FS=OFS="\\t"}
{if(NF==10&&$1~/^[1-9][0-9]*$/||$0~/^# text =/)u=u$0"\\n";
 else if($0==""){f=(FILENAME~/test/)?"test":(FILENAME~/dev/)?"dev":"train";
  if(u~/\\t0\\troot\\t/)print u>f".conllu";
  u=""}
}' $D/*-ud-*.conllu""")
class UDTriangularDataset(object):
  def __init__(self,conllu,tokenizer):
    self.conllu=open(conllu,"r",encoding="utf-8")
    self.tokenizer=tokenizer
    self.seeks=[0]
    label=set(["SYM|x","X|x"])
    dep=set(["X|x|r-goeswith"])
    s=self.conllu.readline()
    while s!="":
      if s=="\n":
        self.seeks.append(self.conllu.tell())
      else:
        w=s.split("\t")
        if len(w)==10:
          if w[0].isdecimal():
            p=w[3]
            q="" if w[5]=="_" else "|"+w[5]
            d=("|" if w[6]=="0" else "|l-" if int(w[0])<int(w[6]) else "|r-")+w[7]
            label.add(p+"|o"+q)
            label.add(p+"|x"+q)
            dep.add(p+"|o"+q+d)
            dep.add(p+"|x"+q+d)
      s=self.conllu.readline()
    lid={l:i for i,l in enumerate(sorted(label))}
    for i,d in enumerate(sorted(dep),len(lid)):
      lid[d]=i
    self.label2id=lid
  def __call__(*args):
    lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))}
    for t in args:
      t.label2id=lid
    return lid
  def __del__(self):
    self.conllu.close()
  __len__=lambda self:len(self.seeks)-1
  def __getitem__(self,i):
    self.conllu.seek(self.seeks[i])
    c,t,s=[],[""],False
    while t[0]!="\n":
      t=self.conllu.readline().split("\t") 
      if len(t)==10 and t[0].isdecimal():
        if s:
           t[1]=" "+t[1]
        c.append(t)
        s=t[9].find("SpaceAfter=No")<0
    v=self.tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
    for i in range(len(v)-1,-1,-1):
      for j in range(1,len(v[i])):
        c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
    y=["0"]+[t[0] for t in c]
    h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
    x=["o" if k>i or sum([1 if j==i+1 else 0 for j in h[i+1:]])>0 else "x" for i,k in enumerate(h)]
    p=[t[3]+"|"+x[i] if t[5]=="_" else t[3]+"|"+x[i]+"|"+t[5] for i,t in enumerate(c)]
    d=[t[7] if t[6]=="0" else "l-"+t[7] if int(t[0])<int(t[6]) else "r-"+t[7] for t in c]
    v=sum(v,[])
    ids=[self.tokenizer.cls_token_id]
    upos=["SYM|x"]
    for i,k in enumerate(v):
      if len(v)<127 or x[i]=="o":
        ids.append(k)
        upos.append(p[i]+"|"+d[i] if h[i]==i+1 else p[i])
        for j in range(i+1,len(v)):
          ids.append(v[j])
          upos.append(p[j]+"|"+d[j] if h[j]==i+1 else p[i]+"|"+d[i] if h[i]==j+1 else p[j])
        ids.append(self.tokenizer.sep_token_id)
        upos.append("SYM|x")
    i=0
    while len(ids)>8192:
      try:
        i=ids.index(self.tokenizer.sep_token_id,ids.index(self.tokenizer.sep_token_id,i+1)+1)-1
      except:
        break
      while len(ids)>8192 and ids[i]!=self.tokenizer.sep_token_id:
        if upos[i].endswith("|x"):
          ids.pop(i)
          upos.pop(i)
          i-=1
        else:
          break
    return {"input_ids":ids[:8192],"labels":[self.label2id[p] for p in upos[:8192]]}
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
tkz=AutoTokenizer.from_pretrained(src)
trainDS=UDTriangularDataset("train.conllu",tkz)
devDS=UDTriangularDataset("dev.conllu",tkz)
testDS=UDTriangularDataset("test.conllu",tkz)
lid=trainDS(devDS,testDS)
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True)
mdl=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True)
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=mdl,train_dataset=trainDS)
trn.train()
trn.save_model(tgt)
tkz.save_pretrained(tgt)