KoichiYasuoka
/

modernbert-small-japanese-aozora-ud-embeds

Token Classification

dependency-parsing

Model card Files Files and versions Community

KoichiYasuoka commited on 4 days ago

Commit

4748fad

·

1 Parent(s): ff5d9cb

bug fix

Files changed (1) hide show

maker.py +2 -2

maker.py CHANGED Viewed

@@ -12,7 +12,7 @@ class UDEmbedsDataset(object):
     self.tokenizer=tokenizer
     self.embeddings=embeddings
     self.seeks=[0]
-    label=set(["SYM","SYM."])
     dep=set()
     s=self.conllu.readline()
     while s!="":
@@ -62,7 +62,7 @@ class UDEmbedsDataset(object):
       emb=self.embeddings
     else:
       import torch
-      if len(x)<128:
         x=[True]*len(x)
       else:
         w=sum([len(x)-i+1 if b else 0 for i,b in enumerate(x)])+1

     self.tokenizer=tokenizer
     self.embeddings=embeddings
     self.seeks=[0]
+    label=set(["SYM","SYM.","SYM|_"])
     dep=set()
     s=self.conllu.readline()
     while s!="":
       emb=self.embeddings
     else:
       import torch
+      if len(x)<127:
         x=[True]*len(x)
       else:
         w=sum([len(x)-i+1 if b else 0 for i,b in enumerate(x)])+1