model update

README.md

@@ -0,0 +1,3 @@
+Reparameterized Absorbing Discrete Diffusion (RADD) small model with t-dce loss. 
+Code: https://github.com/ML-GSAI/RADD. 
+Paper: https://arxiv.org/abs/2406.03736. 

config.json

@@ -0,0 +1,72 @@
+{
+    "ngpus": 32,
+    "tokens": 50257,
+    "gpt_dir": "assets/gpt2-large",
+    "outdir": "../output",
+    "training": {
+        "batch_size": 512,
+        "accum": 1,
+        "n_iters": 1000001,
+        "snapshot_freq": 50000,
+        "log_freq": 50,
+        "eval_freq": 100,
+        "snapshot_freq_for_preemption": 10000,
+        "weight": "standard",
+        "snapshot_sampling": false,
+        "ema": 0.9999,
+        "loss_type": "t_DCE"
+    },
+    "train_set": {
+        "name": "openwebtext",
+        "cache_dir": "assets/datasets/openwebtext"
+    },
+    "valid_set": {
+        "name": "wikitext103",
+        "cache_dir": "assets/datasets/wikitext"
+    },
+    "graph": {
+        "type": "absorb",
+        "file": "data",
+        "report_all": false
+    },
+    "noise": {
+        "type": "loglinear",
+        "sigma_min": 0.0001,
+        "sigma_max": 20
+    },
+    "sampling": {
+        "predictor": "euler",
+        "steps": 128,
+        "noise_removal": true
+    },
+    "eval": {
+        "batch_size": 512,
+        "perplexity": true,
+        "perplexity_batch_size": 16
+    },
+    "optim": {
+        "weight_decay": 0.03,
+        "optimizer": "AdamW",
+        "lr": 0.0003,
+        "beta1": 0.9,
+        "beta2": 0.999,
+        "eps": 1e-08,
+        "warmup": 2500,
+        "grad_clip": 1.0
+    },
+    "model": {
+        "name": "small_wotsm",
+        "type": "ddit_wot",
+        "hidden_size": 768,
+        "cond_dim": 128,
+        "length": 1024,
+        "n_blocks": 12,
+        "n_heads": 12,
+        "scale_by_sigma": true,
+        "dropout": 0.02,
+        "use_checkpoint": false,
+        "remove_time_condition": true,
+        "add_softmax": true,
+        "dtype": "float16"
+    }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0e3d7fbd1284eb9ec9970cdb06b6270bc1cc810d78224e10c67770906021f9c
+size 649074528