model improved

maker.py

@@ -68,7 +68,7 @@ class UDTriangularDataset(object):
     ids=[self.tokenizer.cls_token_id]
     upos=["SYM|x"]
     for i,k in enumerate(v):
-      if len(v)<128 or x[i]=="o":
+      if len(v)<127 or x[i]=="o":
         ids.append(k)
         upos.append(p[i]+"|"+d[i] if h[i]==i+1 else p[i])
         for j in range(i+1,len(v)):
@@ -76,6 +76,19 @@ class UDTriangularDataset(object):
           upos.append(p[j]+"|"+d[j] if h[j]==i+1 else p[i]+"|"+d[i] if h[i]==j+1 else p[j])
         ids.append(self.tokenizer.sep_token_id)
         upos.append("SYM|x")
+    i=0
+    while len(ids)>8192:
+      try:
+        i=ids.index(self.tokenizer.sep_token_id,ids.index(self.tokenizer.sep_token_id,i+1)+1)-1
+      except:
+        break
+      while len(ids)>8192 and ids[i]!=self.tokenizer.sep_token_id:
+        if upos[i].endswith("|x"):
+          ids.pop(i)
+          upos.pop(i)
+          i-=1
+        else:
+          break
     return {"input_ids":ids[:8192],"labels":[self.label2id[p] for p in upos[:8192]]}
 from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
 tkz=AutoTokenizer.from_pretrained(src)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00b384817d170ba11a26b82d40295e13f6b9b9f417143890da581577cdc41851
+oid sha256:0495bee3fdd2fbff26d2bbb1e6368b405767de06ad4de758a198f2aa8b6da608
 size 458220274