Update README.md

README.md

@@ -37,13 +37,19 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TorchAoConfig
 
 model_id = "Qwen/Qwen3-32B"
 
+## Step 1: Convert to float8
 from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
 quant_config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
 quantization_config = TorchAoConfig(quant_type=quant_config)
-quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
+quantized_model = AutoModelForCausalLM.from_pretrained(
+    model_id, 
+    device_map="auto", 
+    torch_dtype=torch.bfloat16, 
+    quantization_config=quantization_config,
+)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
-
+## Step 2: Sanity check
 prompt = "Give me a short introduction to large language model."
 messages = [
     {"role": "user", "content": prompt}
@@ -75,6 +81,14 @@ content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("
 
 print("thinking content:", thinking_content)
 print("content:", content)
+
+
+# Step 3: Upload to HF
+USER_ID = "YOUR_USER_ID"
+MODEL_NAME = model_id.split("/")[-1]
+save_to = f"{USER_ID}/{MODEL_NAME}-float8dq"
+quantized_model.push_to_hub(save_to, safe_serialization=False)
+tokenizer.push_to_hub(save_to)
 ```
 
 # 4. Model Quality