add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Dec02_04-26-18_f44a7de8a132/1638419220.2068994/events.out.tfevents.1638419220.f44a7de8a132.85.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:831d51e3d1bacaed938e55cbda4bb87f8b6fcad7349f11e34506c542e828c7bb
+size 4525

runs/Dec02_04-26-18_f44a7de8a132/events.out.tfevents.1638419220.f44a7de8a132.85.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f6f6250b0411fc9b757bb6cb703f754db4afbe2c2a83adcc1852143e3280d4d
+size 4413

runs/Dec02_04-30-48_f44a7de8a132/1638419476.2656004/events.out.tfevents.1638419476.f44a7de8a132.85.7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:512b958b081cbcdfe20a51d251364537e4c3e4185cc78977035f9deb909a1831
+size 4525

runs/Dec02_04-30-48_f44a7de8a132/events.out.tfevents.1638419476.f44a7de8a132.85.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33beebdff9ab443fabf6712b993ba6c3d1f18277eecf0f930e38bc401e0a19ab
+size 4136