QWEN Chat model for free

.gitignore

@@ -0,0 +1,97 @@
+# skin_color
+.ipynb_checkpoints
+__pycache__/
+tmp/
+flask_session/
+*.sarif
+*.xlsx
+*.h5
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+*.DS_Store
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# PyTorch weights
+*.tar
+*.pth
+*.pt
+*.torch
+*.gz
+Untitled.ipynb
+Testing notebook.ipynb
+
+# Root dir exclusions
+/*.csv
+/*.yaml
+/*.json
+/*.jpg
+/*.png
+/*.zip
+/*.tar.*
+*.jpg
+*.png
+*.avi
+*.mp4
+*.svg
+
+.mypy_cache/
+.vscode/
+.idea
+
+output/
+input/
+
+!test/*
+
+node_modules/
+package-lock.json

QWEN/__init__.py

@@ -0,0 +1,144 @@
+from typing import Optional, List, Any, Dict, Iterator
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
+from langchain_core.outputs import ChatGeneration, ChatResult
+from pydantic import PrivateAttr
+
+# used for qwen inference
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+class ChatQWEN(BaseChatModel):
+    """A custom chat model that invoke Qwen2.5-1.5B-Instruct.
+
+    Example:
+
+        .. code-block:: python
+
+            model = ChatQWEN()
+            result = model.invoke([HumanMessage(content="hello")])
+            result = model.batch([[HumanMessage(content="hello")],
+                                 [HumanMessage(content="world")]])
+    """
+
+    model_name: str = "Qwen/Qwen2.5-1.5B-Instruct"
+    """The name of the model"""
+    # other params
+    temperature: float = 0.7
+    max_new_tokens: int = 512
+    device_map: str = "auto"
+
+    # private attributes
+    _model: Any = PrivateAttr()
+    _tokenizer: Any = PrivateAttr()
+    """The model to call"""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        # load qwen
+        self._tokenizer = AutoTokenizer.from_pretrained(
+            self.model_name, trust_remote_code=True
+        )
+
+        self._model = AutoModelForCausalLM.from_pretrained(
+            self.model_name,
+            device_map=self.device_map,
+            torch_dtype=torch.bfloat16,
+            offload_folder=None,
+            low_cpu_mem_usage=True,
+            trust_remote_code=True,
+        ).eval()
+
+        # Adicione isto após carregar o modelo
+        print(f"GPU memory used: {torch.cuda.memory_allocated()/1024**3:.2f} GB")
+        print(f"GPU memory reserved: {torch.cuda.memory_reserved()/1024**3:.2f} GB")
+
+    def _convert_message_to_dict(self, message: BaseMessage) -> dict:
+        """Messages from LangChain to format expected by QWEN"""
+        if isinstance(message, HumanMessage):
+            return {"role": "user", "content": message.content}
+        elif isinstance(message, AIMessage):
+            return {"role": "assistant", "content": message.content}
+        elif isinstance(message, SystemMessage):
+            return {"role": "system", "content": message.content}
+        else:
+            raise ValueError(f"Message type not supported: {type(message)}")
+
+    def qwen(self, messages):
+        # make the prompt in a way to the model understand
+        text = self._tokenizer.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=True
+        )
+        model_inputs = self._tokenizer([text], return_tensors="pt").to(
+            self._model.device
+        )
+
+        # generate the qwen text
+        with torch.no_grad():
+            generated_ids = self._model.generate(
+                **model_inputs,
+                max_new_tokens=self.max_new_tokens,
+                temperature=self.temperature,
+            )
+            generated_ids = [
+                output_ids[len(input_ids) :]
+                for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+            ]
+
+            # get the response of the LLM
+            response = self._tokenizer.batch_decode(
+                generated_ids, skip_special_tokens=True
+            )[0]
+
+        return response
+
+    def _generate(
+        self,
+        messages: List[BaseMessage],
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[Any] = None,
+        **kwargs: Any,
+    ) -> ChatResult:
+        """
+        Args:
+            messages: the prompt composed of a list of messages.
+        """
+        # parse the messages to feed qwen
+        formatted_messages = [self._convert_message_to_dict(msg) for msg in messages]
+
+        # call qwen
+        qwen_response = self.qwen(formatted_messages)
+
+        # process the stop tokens
+        if stop:
+            for stop_word in stop:
+                qwen_response = qwen_response.split(stop_word)[0]
+
+        # message type update
+        message = AIMessage(content=qwen_response.strip())
+
+        # return
+        generation = ChatGeneration(message=message, text=qwen_response.strip())
+        return ChatResult(generations=[generation])
+
+    @property
+    def _llm_type(self) -> str:
+        """Get the type of language model used by this chat model."""
+        return "qwen-chat-model"
+
+    @property
+    def _identifying_params(self) -> Dict[str, Any]:
+        """Return a dictionary of identifying parameters.
+
+        This information is used by the LangChain callback system, which
+        is used for tracing purposes make it possible to monitor LLMs.
+        """
+        return {
+            # The model name allows users to specify custom token counting
+            # rules in LLM monitoring applications (e.g., in LangSmith users
+            # can provide per token pricing for their model and monitor
+            # costs for the given LLM.)
+            "model_name": self.model_name,
+        }

README.md

@@ -0,0 +1,19 @@
+# Book Reader
+
+## Install
+
+Run the bash script to install the conda environments:
+
+```sh
+bash install.sh
+```
+
+## Run
+
+1. Run the bash script to download the models and run the APIs.
+
+```sh
+bash run.sh
+```
+
+## Example

install.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Path to the conda executable
+CONDA_PATH=$(which conda)
+
+# Check if conda is installed
+if [ -z "$CONDA_PATH" ]; then
+    echo "Conda could not be found. Please install conda first."
+    exit
+fi
+
+# Initialize conda
+eval "$($CONDA_PATH shell.bash hook)"
+
+# List of conda environments to activate
+ENVIRONMENTS=("specialist:3.11")
+
+# Loop through each environment and activate it
+for ENV_NAME_VERSION in "${ENVIRONMENTS[@]}"; do
+    # get environment name and python version
+    IFS=':' read -r ENV_NAME PYTHON_VERSION <<< "$ENV_NAME_VERSION"
+
+    if { conda env list | grep $ENV_NAME; } >/dev/null 2>&1; then
+        echo "$ENV_NAME already exists."
+    else
+        # Create the conda environment
+        echo "Creating conda environment: $ENV_NAME with Python version: $PYTHON_VERSION"
+        conda create -n $ENV_NAME python=$PYTHON_VERSION -y
+    fi
+    # Activate the conda environment
+    echo "Activating conda environment..."
+    conda activate $ENV_NAME
+
+    # Install the requirements from requirements.txt
+    if [ -f requirements.txt ]; then
+        echo "Installing requirements from requirements.txt..."
+        pip install -r requirements.txt
+    else
+        echo "requirements.txt file not found. Please provide the file."
+        exit
+    fi
+
+    echo "Conda environment '$ENV_NAME' is ready."
+
+    echo "Deactivating conda environment..."
+    conda deactivate
+    cd ..
+done
+
+echo "Installation complete."

query.py

@@ -0,0 +1,31 @@
+# chat
+from QWEN import ChatQWEN
+from langchain_core.prompts import ChatPromptTemplate
+
+# prompt chat
+prompt = ChatPromptTemplate(
+    [
+        (
+            "system",
+            "You are Qwen, created by Alibaba Cloud. You are a helpful assistant that translates {input_language} to {output_language}.",
+        ),
+        ("human", "{input}"),
+    ]
+)
+
+# model creation
+llm = ChatQWEN()
+
+# pipeline
+chain = prompt | llm
+
+# query
+print(
+    chain.invoke(
+        {
+            "input_language": "English",
+            "output_language": "German",
+            "input": "I love programming",
+        }
+    ).content
+)

requirements.txt

@@ -0,0 +1,6 @@
+langchain-deepseek==0.1.2
+torch==2.4.1
+triton==3.0.0
+transformers==4.46.3
+safetensors==0.4.5
+accelerate>=0.26.0

run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Path to the conda executable
+CONDA_PATH=$(which conda)
+
+# Check if conda is installed
+if [ -z "$CONDA_PATH" ]; then
+    echo "Conda could not be found. Please install conda first."
+    exit
+fi
+
+# Initialize conda
+eval "$($CONDA_PATH shell.bash hook)"
+
+conda activate specialist
+python query.py
+conda deactivate
+echo "Completed."