WizardCoder-15B-1.0-ggmlv3-q4-0

Build error

App Files Files Community

NERDDISCO

matthoffner commited on Jul 14, 2023

Commit

9eae9de

0 Parent(s):

Duplicate from matthoffner/wizardcoder-ggml

Browse files

Co-authored-by: Matt Hoffner <[email protected]>

Files changed (5) hide show

.gitattributes +34 -0
Dockerfile +23 -0
README.md +21 -0
main.py +128 -0
requirements.txt +12 -0

.gitattributes ADDED Viewed

	@@ -0,0 +1,34 @@

+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile ADDED Viewed

	@@ -0,0 +1,23 @@

+FROM python:latest
+ENV PYTHONUNBUFFERED 1
+EXPOSE 8000
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+COPY requirements.txt ./
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt
+COPY --chown=user . $HOME/app
+RUN ls -al
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md ADDED Viewed

	@@ -0,0 +1,21 @@

+---
+title: wizardcoder
+emoji: 🪄⚡️
+sdk: docker
+app_port: 8000
+duplicated_from: matthoffner/wizardcoder-ggml
+---
+# wizardCoder-ggml
+## <a href="https://matthoffner-wizardcoder-ggml.hf.space/docs" target="_blank">FastAPI Docs</a>
+## <a href="https://github.com/ggerganov/ggml" target="_blank">ggml</a>
+## <a href="https://github.com/marella/ctransformers" target="_blank">ctransformers</a>
+### Updates
+* Refactored /v1/chat/completions to match OpenAI spec
+* Added /v1/chat/completions
+* [Start using ctransformers](https://github.com/marella/ctransformers)
+* [Added starcoder example](https://github.com/ggerganov/ggml/tree/master/examples/starcoder)

main.py ADDED Viewed

	@@ -0,0 +1,128 @@

+import json
+import markdown
+from typing import Callable, List, Dict, Any, Generator
+from functools import partial
+import fastapi
+import uvicorn
+from fastapi import HTTPException, Depends, Request
+from fastapi.responses import HTMLResponse
+from fastapi.middleware.cors import CORSMiddleware
+from sse_starlette.sse import EventSourceResponse
+from anyio import create_memory_object_stream
+from anyio.to_thread import run_sync
+from ctransformers import AutoModelForCausalLM
+from pydantic import BaseModel
+llm = AutoModelForCausalLM.from_pretrained("TheBloke/WizardCoder-15B-1.0-GGML",
+                                           model_file="WizardCoder-15B-1.0.ggmlv3.q5_0.bin",
+                                           model_type="starcoder",
+                                           threads=8)
+app = fastapi.FastAPI(title="🪄WizardCoder💫")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+@app.get("/")
+async def index():
+    html_content = """
+    <html>
+        <head>
+        </head>
+        <body style="font-family:system-ui">
+            <h2><a href="https://huggingface.co/TheBloke/WizardCoder-15B-1.0-GGML">wizardcoder-ggml</a></h2>
+            <h2><a href="https://matthoffner-wizardcoder-ggml.hf.space/docs">FastAPI Docs</a></h2>
+            <h2><a href="https://wizardcoder-sandbox.netlify.app">Wizardcoder Sandbox</a></h2>
+            <h2><a href="https://matthoffner-monacopilot.hf.space">monacopilot</a></h2>
+        </body>
+    </html>
+    """
+    return HTMLResponse(content=html_content, status_code=200)
+class ChatCompletionRequestV0(BaseModel):
+    prompt: str
+class Message(BaseModel):
+    role: str
+    content: str
+class ChatCompletionRequest(BaseModel):
+    messages: List[Message]
+    max_tokens: int = 250
+@app.post("/v1/completions")
+async def completion(request: ChatCompletionRequestV0, response_mode=None):
+    response = llm(request.prompt)
+    return response
+async def generate_response(chat_chunks, llm):
+    for chat_chunk in chat_chunks:
+        response = {
+            'choices': [
+                {
+                    'message': {
+                        'role': 'system',
+                        'content': llm.detokenize(chat_chunk)
+                    },
+                    'finish_reason': 'stop' if llm.is_eos_token(chat_chunk) else 'unknown'
+                }
+            ]
+        }
+        yield dict(data=json.dumps(response))
+    yield dict(data="[DONE]")
+@app.post("/v1/chat/completions")
+async def chat(request: ChatCompletionRequest):
+    combined_messages = ' '.join([message.content for message in request.messages])
+    tokens = llm.tokenize(combined_messages)
+    try:
+        chat_chunks = llm.generate(tokens)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    return EventSourceResponse(generate_response(chat_chunks, llm))
+async def stream_response(tokens, llm):
+    try:
+        iterator: Generator = llm.generate(tokens)
+        for chat_chunk in iterator:
+            response = {
+                'choices': [
+                    {
+                        'message': {
+                            'role': 'system',
+                            'content': llm.detokenize(chat_chunk)
+                        },
+                        'finish_reason': 'stop' if llm.is_eos_token(chat_chunk) else 'unknown'
+                    }
+                ]
+            }
+            yield dict(data=json.dumps(response))
+        yield dict(data="[DONE]")
+    except Exception as e:
+        print(f"Exception in event publisher: {str(e)}")
+@app.post("/v2/chat/completions")
+async def chatV2_endpoint(request: Request, body: ChatCompletionRequest):
+    combined_messages = ' '.join([message.content for message in body.messages])
+    tokens = llm.tokenize(combined_messages)
+    return EventSourceResponse(stream_response(tokens, llm))
+@app.post("/v0/chat/completions")
+async def chat(request: ChatCompletionRequestV0, response_mode=None):
+    tokens = llm.tokenize(request.prompt)
+    async def server_sent_events(chat_chunks, llm):
+        for chat_chunk in llm.generate(chat_chunks):
+            yield dict(data=json.dumps(llm.detokenize(chat_chunk)))
+        yield dict(data="[DONE]")
+    return EventSourceResponse(server_sent_events(tokens, llm))
+if __name__ == "__main__":
+  uvicorn.run(app, host="0.0.0.0", port=8000)

requirements.txt ADDED Viewed

	@@ -0,0 +1,12 @@

+uvicorn
+markdown
+fastapi
+loguru
+torch
+numpy
+transformers
+ctransformers
+accelerate
+langchain
+sse_starlette
+anyio