initial commit

.gitignore

@@ -0,0 +1 @@
+.env

README.md

@@ -0,0 +1,50 @@
+---
+language:
+  - en
+tags:
+  - text-generation
+  - llama
+  - instruct
+license: apache-2.0
+pipeline_tag: text-generation
+---
+
+# LLaDA-8B-Instruct Model
+
+This is the LLaDA-8B-Instruct model deployed as a Hugging Face inference endpoint.
+
+## Model Details
+
+LLaDA-8B-Instruct is a language model designed for instruction-following tasks.
+
+## Usage
+
+This model can be used for text generation tasks. Here's an example:
+
+```python
+import requests
+
+API_URL = "https://YOUR-ENDPOINT-URL"
+headers = {"Authorization": "Bearer YOUR_HF_TOKEN"}
+
+def query(payload):
+    response = requests.post(API_URL, headers=headers, json=payload)
+    return response.json()
+
+output = query({
+    "inputs": "Write a short story about a robot learning to paint:",
+    "parameters": {
+        "max_new_tokens": 250,
+        "temperature": 0.7,
+        "top_p": 0.95,
+        "do_sample": true
+    }
+})
+```
+
+## API Inference Configuration
+
+```yaml
+api_inference:
+  handler_class: handler.EndpointHandler
+```

handler.py

@@ -0,0 +1,67 @@
+from typing import Dict, List, Any
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # Load model with half precision to save memory
+        self.model = AutoModelForCausalLM.from_pretrained(
+            path, torch_dtype=torch.float16, device_map="auto"
+        )
+
+        # Load tokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+
+        # Ensure pad token is properly set
+        if self.tokenizer.pad_token_id is None:
+            if (
+                hasattr(self.tokenizer, "eos_token_id")
+                and self.tokenizer.eos_token_id is not None
+            ):
+                self.tokenizer.pad_token_id = self.tokenizer.eos_token_id
+                self.tokenizer.pad_token = self.tokenizer.eos_token
+            else:
+                # Fallback to a common pad token
+                self.tokenizer.pad_token_id = 0
+                self.tokenizer.pad_token = self.tokenizer.convert_ids_to_tokens(0)
+
+        print(f"Model loaded successfully. Pad token ID: {self.tokenizer.pad_token_id}")
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, List[Any]]:
+        """Handle inference requests"""
+        # Extract inputs and parameters from request data
+        inputs = data.pop("inputs", data)
+        parameters = data.pop("parameters", {})
+
+        # Convert single string input to list for consistent handling
+        if isinstance(inputs, str):
+            inputs = [inputs]
+
+        # Extract generation parameters with sensible defaults
+        max_new_tokens = parameters.get("max_new_tokens", 256)
+        temperature = parameters.get("temperature", 0.7)
+        top_p = parameters.get("top_p", 0.95)
+        do_sample = parameters.get("do_sample", True)
+
+        # Tokenize inputs
+        input_tokens = self.tokenizer(inputs, return_tensors="pt", padding=True).to(
+            self.model.device
+        )
+
+        # Generate text
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **input_tokens,
+                max_new_tokens=max_new_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                do_sample=do_sample,
+                pad_token_id=self.tokenizer.pad_token_id,
+            )
+
+        # Decode generated text
+        generated_texts = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
+
+        # Return results in expected format
+        return {"generated_text": generated_texts}