Source code for langchain.llms.symblai_nebula
import logging
from typing import Any, Dict, List, Mapping, Optional
import requests
from pydantic import Extra, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.utils import get_from_dict_or_env
DEFAULT_NEBULA_SERVICE_URL = "https://api-nebula.symbl.ai"
DEFAULT_NEBULA_SERVICE_PATH = "/v1/model/generate"
logger = logging.getLogger(__name__)
[docs]class Nebula(LLM):
    """Nebula Service models.
    To use, you should have the environment variable ``NEBULA_SERVICE_URL``,
    ``NEBULA_SERVICE_PATH`` and ``NEBULA_SERVICE_API_KEY`` set with your Nebula
    Service, or pass it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain.llms import Nebula
            nebula = Nebula(
                nebula_service_url="SERVICE_URL",
                nebula_service_path="SERVICE_ROUTE",
                nebula_api_key="SERVICE_TOKEN",
            )
    """  # noqa: E501
    """Key/value arguments to pass to the model. Reserved for future use"""
    model_kwargs: Optional[dict] = None
    """Optional"""
    nebula_service_url: Optional[str] = None
    nebula_service_path: Optional[str] = None
    nebula_api_key: Optional[str] = None
    conversation: str = ""
    return_scores: Optional[str] = "false"
    max_new_tokens: Optional[int] = 2048
    top_k: Optional[float] = 2
    penalty_alpha: Optional[float] = 0.1


penalty_alpha: Optional[float] = 0.1
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        nebula_service_url = get_from_dict_or_env(
            values,
            "nebula_service_url",
            "NEBULA_SERVICE_URL",
            DEFAULT_NEBULA_SERVICE_URL,
        )
        nebula_service_path = get_from_dict_or_env(
            values,
            "nebula_service_path",
            "NEBULA_SERVICE_PATH",
            DEFAULT_NEBULA_SERVICE_PATH,
        )
        nebula_api_key = get_from_dict_or_env(
            values, "nebula_api_key", "NEBULA_SERVICE_API_KEY", ""
        )
        if nebula_service_url.endswith("/"):
            nebula_service_url = nebula_service_url[:-1]
        if not nebula_service_path.startswith("/"):
            nebula_service_path = "/" + nebula_service_path
        """ TODO: Future login"""
        """
        try:
            nebula_service_endpoint = f"{nebula_service_url}{nebula_service_path}"
            headers = {
                "Content-Type": "application/json",
                "ApiKey": "{nebula_api_key}",
                }
            requests.get(nebula_service_endpoint, headers=headers)
        except requests.exceptions.RequestException as e:
            raise ValueError(e)
        """
        values["nebula_service_url"] = nebula_service_url
        values["nebula_service_path"] = nebula_service_path
        values["nebula_api_key"] = nebula_api_key
        return values
    @property


return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        _model_kwargs = self.model_kwargs or {}
        return {
            "nebula_service_url": self.nebula_service_url,
            "nebula_service_path": self.nebula_service_path,
            **{"model_kwargs": _model_kwargs},
            "conversation": self.conversation,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "nebula"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Nebula Service endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = nebula("Tell me a joke.")
        """
        _model_kwargs = self.model_kwargs or {}
        nebula_service_endpoint = f"{self.nebula_service_url}{self.nebula_service_path}"
        headers = {
            "Content-Type": "application/json",
            "ApiKey": f"{self.nebula_api_key}",
        }
        body = {
            "prompt": {
                "instruction": prompt,
                "conversation": {"text": f"{self.conversation}"},
            },
            "return_scores": self.return_scores,
            "max_new_tokens": self.max_new_tokens,


"max_new_tokens": self.max_new_tokens,
            "top_k": self.top_k,
            "penalty_alpha": self.penalty_alpha,
        }
        if len(self.conversation) == 0:
            raise ValueError("Error conversation is empty.")
        logger.debug(f"NEBULA _model_kwargs: {_model_kwargs}")
        logger.debug(f"NEBULA body: {body}")
        logger.debug(f"NEBULA kwargs: {kwargs}")
        logger.debug(f"NEBULA conversation: {self.conversation}")
        # call API
        try:
            response = requests.post(
                nebula_service_endpoint, headers=headers, json=body
            )
        except requests.exceptions.RequestException as e:
            raise ValueError(f"Error raised by inference endpoint: {e}")
        logger.debug(f"NEBULA response: {response}")
        if response.status_code != 200:
            raise ValueError(
                f"Error returned by service, status code {response.status_code}"
            )
        """ get the result """
        text = response.text
        """ enforce stop """
        if stop is not None:
            # This is required since the stop tokens
            # are not enforced by the model parameters
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.promptlayer_openai
import datetime
from typing import Any, List, Optional
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms import OpenAI, OpenAIChat
from langchain.schema import LLMResult
[docs]class PromptLayerOpenAI(OpenAI):
    """PromptLayer OpenAI large language models.
    To use, you should have the ``openai`` and ``promptlayer`` python
    package installed, and the environment variable ``OPENAI_API_KEY``
    and ``PROMPTLAYER_API_KEY`` set with your openAI API key and
    promptlayer key respectively.
    All parameters that can be passed to the OpenAI LLM can also
    be passed here. The PromptLayerOpenAI LLM adds two optional
    parameters:
        ``pl_tags``: List of strings to tag the request with.
        ``return_pl_id``: If True, the PromptLayer request ID will be
            returned in the ``generation_info`` field of the
            ``Generation`` object.
    Example:
        .. code-block:: python
            from langchain.llms import PromptLayerOpenAI
            openai = PromptLayerOpenAI(model_name="text-davinci-003")
    """
    pl_tags: Optional[List[str]]
    return_pl_id: Optional[bool] = False
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        """Call OpenAI generate and then call PromptLayer API to log the request."""


"""Call OpenAI generate and then call PromptLayer API to log the request."""
        from promptlayer.utils import get_api_key, promptlayer_api_request
        request_start_time = datetime.datetime.now().timestamp()
        generated_responses = super()._generate(prompts, stop, run_manager)
        request_end_time = datetime.datetime.now().timestamp()
        for i in range(len(prompts)):
            prompt = prompts[i]
            generation = generated_responses.generations[i][0]
            resp = {
                "text": generation.text,
                "llm_output": generated_responses.llm_output,
            }
            params = {**self._identifying_params, **kwargs}
            pl_request_id = promptlayer_api_request(
                "langchain.PromptLayerOpenAI",
                "langchain",
                [prompt],
                params,
                self.pl_tags,
                resp,
                request_start_time,
                request_end_time,
                get_api_key(),
                return_pl_id=self.return_pl_id,
            )
            if self.return_pl_id:
                if generation.generation_info is None or not isinstance(
                    generation.generation_info, dict
                ):
                    generation.generation_info = {}
                generation.generation_info["pl_request_id"] = pl_request_id
        return generated_responses
    async def _agenerate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        from promptlayer.utils import get_api_key, promptlayer_api_request_async
        request_start_time = datetime.datetime.now().timestamp()
        generated_responses = await super()._agenerate(prompts, stop, run_manager)


generated_responses = await super()._agenerate(prompts, stop, run_manager)
        request_end_time = datetime.datetime.now().timestamp()
        for i in range(len(prompts)):
            prompt = prompts[i]
            generation = generated_responses.generations[i][0]
            resp = {
                "text": generation.text,
                "llm_output": generated_responses.llm_output,
            }
            params = {**self._identifying_params, **kwargs}
            pl_request_id = await promptlayer_api_request_async(
                "langchain.PromptLayerOpenAI.async",
                "langchain",
                [prompt],
                params,
                self.pl_tags,
                resp,
                request_start_time,
                request_end_time,
                get_api_key(),
                return_pl_id=self.return_pl_id,
            )
            if self.return_pl_id:
                if generation.generation_info is None or not isinstance(
                    generation.generation_info, dict
                ):
                    generation.generation_info = {}
                generation.generation_info["pl_request_id"] = pl_request_id
        return generated_responses
[docs]class PromptLayerOpenAIChat(OpenAIChat):
    """Wrapper around OpenAI large language models.
    To use, you should have the ``openai`` and ``promptlayer`` python
    package installed, and the environment variable ``OPENAI_API_KEY``
    and ``PROMPTLAYER_API_KEY`` set with your openAI API key and
    promptlayer key respectively.
    All parameters that can be passed to the OpenAIChat LLM can also
    be passed here. The PromptLayerOpenAIChat adds two optional
    parameters:
        ``pl_tags``: List of strings to tag the request with.


parameters:
        ``pl_tags``: List of strings to tag the request with.
        ``return_pl_id``: If True, the PromptLayer request ID will be
            returned in the ``generation_info`` field of the
            ``Generation`` object.
    Example:
        .. code-block:: python
            from langchain.llms import PromptLayerOpenAIChat
            openaichat = PromptLayerOpenAIChat(model_name="gpt-3.5-turbo")
    """
    pl_tags: Optional[List[str]]
    return_pl_id: Optional[bool] = False
    def _generate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        """Call OpenAI generate and then call PromptLayer API to log the request."""
        from promptlayer.utils import get_api_key, promptlayer_api_request
        request_start_time = datetime.datetime.now().timestamp()
        generated_responses = super()._generate(prompts, stop, run_manager)
        request_end_time = datetime.datetime.now().timestamp()
        for i in range(len(prompts)):
            prompt = prompts[i]
            generation = generated_responses.generations[i][0]
            resp = {
                "text": generation.text,
                "llm_output": generated_responses.llm_output,
            }
            params = {**self._identifying_params, **kwargs}
            pl_request_id = promptlayer_api_request(
                "langchain.PromptLayerOpenAIChat",
                "langchain",
                [prompt],
                params,
                self.pl_tags,
                resp,
                request_start_time,
                request_end_time,


resp,
                request_start_time,
                request_end_time,
                get_api_key(),
                return_pl_id=self.return_pl_id,
            )
            if self.return_pl_id:
                if generation.generation_info is None or not isinstance(
                    generation.generation_info, dict
                ):
                    generation.generation_info = {}
                generation.generation_info["pl_request_id"] = pl_request_id
        return generated_responses
    async def _agenerate(
        self,
        prompts: List[str],
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> LLMResult:
        from promptlayer.utils import get_api_key, promptlayer_api_request_async
        request_start_time = datetime.datetime.now().timestamp()
        generated_responses = await super()._agenerate(prompts, stop, run_manager)
        request_end_time = datetime.datetime.now().timestamp()
        for i in range(len(prompts)):
            prompt = prompts[i]
            generation = generated_responses.generations[i][0]
            resp = {
                "text": generation.text,
                "llm_output": generated_responses.llm_output,
            }
            params = {**self._identifying_params, **kwargs}
            pl_request_id = await promptlayer_api_request_async(
                "langchain.PromptLayerOpenAIChat.async",
                "langchain",
                [prompt],
                params,
                self.pl_tags,
                resp,
                request_start_time,
                request_end_time,
                get_api_key(),
                return_pl_id=self.return_pl_id,
            )
            if self.return_pl_id:
                if generation.generation_info is None or not isinstance(
                    generation.generation_info, dict


generation.generation_info, dict
                ):
                    generation.generation_info = {}
                generation.generation_info["pl_request_id"] = pl_request_id
        return generated_responses


Source code for langchain.llms.self_hosted
import importlib.util
import logging
import pickle
from typing import Any, Callable, List, Mapping, Optional
from pydantic import Extra
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
logger = logging.getLogger(__name__)
def _generate_text(
    pipeline: Any,
    prompt: str,
    *args: Any,
    stop: Optional[List[str]] = None,
    **kwargs: Any,
) -> str:
    """Inference function to send to the remote hardware.
    Accepts a pipeline callable (or, more likely,
    a key pointing to the model on the cluster's object store)
    and returns text predictions for each document
    in the batch.
    """
    text = pipeline(prompt, *args, **kwargs)
    if stop is not None:
        text = enforce_stop_tokens(text, stop)
    return text
def _send_pipeline_to_device(pipeline: Any, device: int) -> Any:
    """Send a pipeline to a device on the cluster."""
    if isinstance(pipeline, str):
        with open(pipeline, "rb") as f:
            pipeline = pickle.load(f)
    if importlib.util.find_spec("torch") is not None:
        import torch
        cuda_device_count = torch.cuda.device_count()
        if device < -1 or (device >= cuda_device_count):
            raise ValueError(
                f"Got device=={device}, "
                f"device is required to be within [-1, {cuda_device_count})"
            )
        if device < 0 and cuda_device_count > 0:
            logger.warning(


if device < 0 and cuda_device_count > 0:
            logger.warning(
                "Device has %d GPUs available. "
                "Provide device={deviceId} to `from_model_id` to use available"
                "GPUs for execution. deviceId is -1 for CPU and "
                "can be a positive integer associated with CUDA device id.",
                cuda_device_count,
            )
        pipeline.device = torch.device(device)
        pipeline.model = pipeline.model.to(pipeline.device)
    return pipeline
[docs]class SelfHostedPipeline(LLM):
    """Model inference on self-hosted remote hardware.
    Supported hardware includes auto-launched instances on AWS, GCP, Azure,
    and Lambda, as well as servers specified
    by IP address and SSH credentials (such as on-prem, or another
    cloud like Paperspace, Coreweave, etc.).
    To use, you should have the ``runhouse`` python package installed.
    Example for custom pipeline and inference functions:
        .. code-block:: python
            from langchain.llms import SelfHostedPipeline
            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
            import runhouse as rh
            def load_pipeline():
                tokenizer = AutoTokenizer.from_pretrained("gpt2")
                model = AutoModelForCausalLM.from_pretrained("gpt2")
                return pipeline(
                    "text-generation", model=model, tokenizer=tokenizer,
                    max_new_tokens=10
                )
            def inference_fn(pipeline, prompt, stop = None):
                return pipeline(prompt)[0]["generated_text"]
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            llm = SelfHostedPipeline(
                model_load_fn=load_pipeline,
                hardware=gpu,


model_load_fn=load_pipeline,
                hardware=gpu,
                model_reqs=model_reqs, inference_fn=inference_fn
            )
    Example for <2GB model (can be serialized and sent directly to the server):
        .. code-block:: python
            from langchain.llms import SelfHostedPipeline
            import runhouse as rh
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            my_model = ...
            llm = SelfHostedPipeline.from_pipeline(
                pipeline=my_model,
                hardware=gpu,
                model_reqs=["./", "torch", "transformers"],
            )
    Example passing model path for larger models:
        .. code-block:: python
            from langchain.llms import SelfHostedPipeline
            import runhouse as rh
            import pickle
            from transformers import pipeline
            generator = pipeline(model="gpt2")
            rh.blob(pickle.dumps(generator), path="models/pipeline.pkl"
                ).save().to(gpu, path="models")
            llm = SelfHostedPipeline.from_pipeline(
                pipeline="models/pipeline.pkl",
                hardware=gpu,
                model_reqs=["./", "torch", "transformers"],
            )
    """
    pipeline_ref: Any  #: :meta private:
    client: Any  #: :meta private:
    inference_fn: Callable = _generate_text  #: :meta private:
    """Inference function to send to the remote hardware."""
    hardware: Any
    """Remote hardware to send the inference function to."""
    model_load_fn: Callable
    """Function to load the model remotely on the server."""
    load_fn_kwargs: Optional[dict] = None
    """Key word arguments to pass to the model load function."""


"""Key word arguments to pass to the model load function."""
    model_reqs: List[str] = ["./", "torch"]
    """Requirements to install on hardware to inference the model."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    def __init__(self, **kwargs: Any):
        """Init the pipeline with an auxiliary function.
        The load function must be in global scope to be imported
        and run on the server, i.e. in a module and not a REPL or closure.
        Then, initialize the remote inference function.
        """
        super().__init__(**kwargs)
        try:
            import runhouse as rh
        except ImportError:
            raise ImportError(
                "Could not import runhouse python package. "
                "Please install it with `pip install runhouse`."
            )
        remote_load_fn = rh.function(fn=self.model_load_fn).to(
            self.hardware, reqs=self.model_reqs
        )
        _load_fn_kwargs = self.load_fn_kwargs or {}
        self.pipeline_ref = remote_load_fn.remote(**_load_fn_kwargs)
        self.client = rh.function(fn=self.inference_fn).to(
            self.hardware, reqs=self.model_reqs
        )
[docs]    @classmethod
    def from_pipeline(
        cls,
        pipeline: Any,
        hardware: Any,
        model_reqs: Optional[List[str]] = None,
        device: int = 0,
        **kwargs: Any,
    ) -> LLM:
        """Init the SelfHostedPipeline from a pipeline object or string."""
        if not isinstance(pipeline, str):
            logger.warning(
                "Serializing pipeline to send to remote hardware. "


logger.warning(
                "Serializing pipeline to send to remote hardware. "
                "Note, it can be quite slow"
                "to serialize and send large models with each execution. "
                "Consider sending the pipeline"
                "to the cluster and passing the path to the pipeline instead."
            )
        load_fn_kwargs = {"pipeline": pipeline, "device": device}
        return cls(
            load_fn_kwargs=load_fn_kwargs,
            model_load_fn=_send_pipeline_to_device,
            hardware=hardware,
            model_reqs=["transformers", "torch"] + (model_reqs or []),
            **kwargs,
        )
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"hardware": self.hardware},
        }
    @property
    def _llm_type(self) -> str:
        return "self_hosted_llm"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        return self.client(
            pipeline=self.pipeline_ref, prompt=prompt, stop=stop, **kwargs
        )


Source code for langchain.llms.baseten
import logging
from typing import Any, Dict, List, Mapping, Optional
from pydantic import Field
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
logger = logging.getLogger(__name__)
[docs]class Baseten(LLM):
    """Baseten models.
    To use, you should have the ``baseten`` python package installed,
    and run ``baseten.login()`` with your Baseten API key.
    The required ``model`` param can be either a model id or model
    version id. Using a model version ID will result in
    slightly faster invocation.
    Any other model parameters can also
    be passed in with the format input={model_param: value, ...}
    The Baseten model must accept a dictionary of input with the key
    "prompt" and return a dictionary with a key "data" which maps
    to a list of response strings.
    Example:
        .. code-block:: python
            from langchain.llms import Baseten
            my_model = Baseten(model="MODEL_ID")
            output = my_model("prompt")
    """
    model: str
    input: Dict[str, Any] = Field(default_factory=dict)
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"model_kwargs": self.model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        """Return type of model."""
        return "baseten"
    def _call(
        self,
        prompt: str,


def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call to Baseten deployed model endpoint."""
        try:
            import baseten
        except ImportError as exc:
            raise ImportError(
                "Could not import Baseten Python package. "
                "Please install it with `pip install baseten`."
            ) from exc
        # get the model and version
        try:
            model = baseten.deployed_model_version_id(self.model)
            response = model.predict({"prompt": prompt, **kwargs})
        except baseten.common.core.ApiError:
            model = baseten.deployed_model_id(self.model)
            response = model.predict({"prompt": prompt, **kwargs})
        return "".join(response)


Source code for langchain.llms.bananadev
import logging
from typing import Any, Dict, List, Mapping, Optional
from pydantic import Extra, Field, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class Banana(LLM):
    """Banana large language models.
    To use, you should have the ``banana-dev`` python package installed,
    and the environment variable ``BANANA_API_KEY`` set with your API key.
    Any parameters that are valid to be passed to the call can be passed
    in, even if not explicitly saved on this class.
    Example:
        .. code-block:: python
            from langchain.llms import Banana
            banana = Banana(model_key="")
    """
    model_key: str = ""
    """model endpoint to use"""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not
    explicitly specified."""
    banana_api_key: Optional[str] = None
    class Config:
        """Configuration for this pydantic config."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:


if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""{field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        banana_api_key = get_from_dict_or_env(
            values, "banana_api_key", "BANANA_API_KEY"
        )
        values["banana_api_key"] = banana_api_key
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"model_key": self.model_key},
            **{"model_kwargs": self.model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "bananadev"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call to Banana endpoint."""
        try:
            import banana_dev as banana
        except ImportError:
            raise ImportError(
                "Could not import banana-dev python package. "
                "Please install it with `pip install banana-dev`."
            )
        params = self.model_kwargs or {}


)
        params = self.model_kwargs or {}
        params = {**params, **kwargs}
        api_key = self.banana_api_key
        model_key = self.model_key
        model_inputs = {
            # a json specific to your model.
            "prompt": prompt,
            **params,
        }
        response = banana.run(api_key, model_key, model_inputs)
        try:
            text = response["modelOutputs"][0]["output"]
        except (KeyError, TypeError):
            returned = response["modelOutputs"][0]
            raise ValueError(
                "Response should be of schema: {'output': 'text'}."
                f"\nResponse was: {returned}"
                "\nTo fix this:"
                "\n- fork the source repo of the Banana model"
                "\n- modify app.py to return the above schema"
                "\n- deploy that as a custom repo"
            )
        if stop is not None:
            # I believe this is required since the stop tokens
            # are not enforced by the model parameters
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.modal
import logging
from typing import Any, Dict, List, Mapping, Optional
import requests
from pydantic import Extra, Field, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
logger = logging.getLogger(__name__)
[docs]class Modal(LLM):
    """Modal large language models.
    To use, you should have the ``modal-client`` python package installed.
    Any parameters that are valid to be passed to the call can be passed
    in, even if not explicitly saved on this class.
    Example:
        .. code-block:: python
            from langchain.llms import Modal
            modal = Modal(endpoint_url="")
    """
    endpoint_url: str = ""
    """model endpoint to use"""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not
    explicitly specified."""
    class Config:
        """Configuration for this pydantic config."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""{field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""


Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"endpoint_url": self.endpoint_url},
            **{"model_kwargs": self.model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "modal"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call to Modal endpoint."""
        params = self.model_kwargs or {}
        params = {**params, **kwargs}
        response = requests.post(
            url=self.endpoint_url,
            headers={
                "Content-Type": "application/json",
            },
            json={"prompt": prompt, **params},
        )
        try:
            if prompt in response.json()["prompt"]:
                response_json = response.json()
        except KeyError:
            raise ValueError("LangChain requires 'prompt' key in response.")
        text = response_json["prompt"]
        if stop is not None:
            # I believe this is required since the stop tokens
            # are not enforced by the model parameters
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.loading
"""Base interface for loading large language model APIs."""
import json
from pathlib import Path
from typing import Union
import yaml
from langchain.llms import type_to_cls_dict
from langchain.llms.base import BaseLLM
[docs]def load_llm_from_config(config: dict) -> BaseLLM:
    """Load LLM from Config Dict."""
    if "_type" not in config:
        raise ValueError("Must specify an LLM Type in config")
    config_type = config.pop("_type")
    if config_type not in type_to_cls_dict:
        raise ValueError(f"Loading {config_type} LLM not supported")
    llm_cls = type_to_cls_dict[config_type]
    return llm_cls(**config)
[docs]def load_llm(file: Union[str, Path]) -> BaseLLM:
    """Load LLM from file."""
    # Convert file to Path object.
    if isinstance(file, str):
        file_path = Path(file)
    else:
        file_path = file
    # Load from either json or yaml.
    if file_path.suffix == ".json":
        with open(file_path) as f:
            config = json.load(f)
    elif file_path.suffix == ".yaml":
        with open(file_path, "r") as f:
            config = yaml.safe_load(f)
    else:
        raise ValueError("File type must be json or yaml")
    # Load the LLM from the config now.
    return load_llm_from_config(config)


Source code for langchain.llms.manifest
from typing import Any, Dict, List, Mapping, Optional
from pydantic import Extra, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
[docs]class ManifestWrapper(LLM):
    """HazyResearch's Manifest library."""
    client: Any  #: :meta private:
    llm_kwargs: Optional[Dict] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that python package exists in environment."""
        try:
            from manifest import Manifest
            if not isinstance(values["client"], Manifest):
                raise ValueError
        except ImportError:
            raise ImportError(
                "Could not import manifest python package. "
                "Please install it with `pip install manifest-ml`."
            )
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        kwargs = self.llm_kwargs or {}
        return {**self.client.client.get_model_params(), **kwargs}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "manifest"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to LLM through Manifest."""
        if stop is not None and len(stop) != 1:
            raise NotImplementedError(
                f"Manifest currently only supports a single stop token, got {stop}"


f"Manifest currently only supports a single stop token, got {stop}"
            )
        params = self.llm_kwargs or {}
        params = {**params, **kwargs}
        if stop is not None:
            params["stop_token"] = stop
        return self.client.run(prompt, **params)


Source code for langchain.llms.mosaicml
from typing import Any, Dict, List, Mapping, Optional
import requests
from pydantic import Extra, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.utils import get_from_dict_or_env
INSTRUCTION_KEY = "### Instruction:"
RESPONSE_KEY = "### Response:"
INTRO_BLURB = (
    "Below is an instruction that describes a task. "
    "Write a response that appropriately completes the request."
)
PROMPT_FOR_GENERATION_FORMAT = """{intro}
{instruction_key}
{instruction}
{response_key}
""".format(
    intro=INTRO_BLURB,
    instruction_key=INSTRUCTION_KEY,
    instruction="{instruction}",
    response_key=RESPONSE_KEY,
)
[docs]class MosaicML(LLM):
    """MosaicML LLM service.
    To use, you should have the
    environment variable ``MOSAICML_API_TOKEN`` set with your API token, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain.llms import MosaicML
            endpoint_url = (
                "https://models.hosted-on.mosaicml.hosting/mpt-7b-instruct/v1/predict"
            )
            mosaic_llm = MosaicML(
                endpoint_url=endpoint_url,
                mosaicml_api_token="my-api-key"
            )
    """
    endpoint_url: str = (
        "https://models.hosted-on.mosaicml.hosting/mpt-7b-instruct/v1/predict"
    )
    """Endpoint URL to use."""


)
    """Endpoint URL to use."""
    inject_instruction_format: bool = False
    """Whether to inject the instruction format into the prompt."""
    model_kwargs: Optional[dict] = None
    """Key word arguments to pass to the model."""
    retry_sleep: float = 1.0
    """How long to try sleeping for if a rate limit is encountered"""
    mosaicml_api_token: Optional[str] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        mosaicml_api_token = get_from_dict_or_env(
            values, "mosaicml_api_token", "MOSAICML_API_TOKEN"
        )
        values["mosaicml_api_token"] = mosaicml_api_token
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        _model_kwargs = self.model_kwargs or {}
        return {
            **{"endpoint_url": self.endpoint_url},
            **{"model_kwargs": _model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "mosaic"
    def _transform_prompt(self, prompt: str) -> str:
        """Transform prompt."""
        if self.inject_instruction_format:
            prompt = PROMPT_FOR_GENERATION_FORMAT.format(
                instruction=prompt,
            )
        return prompt
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,


prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        is_retry: bool = False,
        **kwargs: Any,
    ) -> str:
        """Call out to a MosaicML LLM inference endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = mosaic_llm("Tell me a joke.")
        """
        _model_kwargs = self.model_kwargs or {}
        prompt = self._transform_prompt(prompt)
        payload = {"inputs": [prompt]}
        payload.update(_model_kwargs)
        payload.update(kwargs)
        # HTTP headers for authorization
        headers = {
            "Authorization": f"{self.mosaicml_api_token}",
            "Content-Type": "application/json",
        }
        # send request
        try:
            response = requests.post(self.endpoint_url, headers=headers, json=payload)
        except requests.exceptions.RequestException as e:
            raise ValueError(f"Error raised by inference endpoint: {e}")
        try:
            parsed_response = response.json()
            if "error" in parsed_response:
                # if we get rate limited, try sleeping for 1 second
                if (
                    not is_retry
                    and "rate limit exceeded" in parsed_response["error"].lower()
                ):
                    import time
                    time.sleep(self.retry_sleep)
                    return self._call(prompt, stop, run_manager, is_retry=True)
                raise ValueError(
                    f"Error raised by inference API: {parsed_response['error']}"
                )


f"Error raised by inference API: {parsed_response['error']}"
                )
            # The inference API has changed a couple of times, so we add some handling
            # to be robust to multiple response formats.
            if isinstance(parsed_response, dict):
                output_keys = ["data", "output", "outputs"]
                for key in output_keys:
                    if key in parsed_response:
                        output_item = parsed_response[key]
                        break
                else:
                    raise ValueError(
                        f"No valid key ({', '.join(output_keys)}) in response:"
                        f" {parsed_response}"
                    )
                if isinstance(output_item, list):
                    text = output_item[0]
                else:
                    text = output_item
            elif isinstance(parsed_response, list):
                first_item = parsed_response[0]
                if isinstance(first_item, str):
                    text = first_item
                elif isinstance(first_item, dict):
                    if "output" in parsed_response:
                        text = first_item["output"]
                    else:
                        raise ValueError(
                            f"No key data or output in response: {parsed_response}"
                        )
                else:
                    raise ValueError(f"Unexpected response format: {parsed_response}")
            else:
                raise ValueError(f"Unexpected response type: {parsed_response}")
            text = text[len(prompt) :]
        except requests.exceptions.JSONDecodeError as e:
            raise ValueError(
                f"Error raised by inference API: {e}.\nResponse: {response.text}"
            )
        # TODO: replace when MosaicML supports custom stop tokens natively
        if stop is not None:
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.aviary
import dataclasses
import os
from typing import Any, Dict, List, Mapping, Optional, Union, cast
import requests
from pydantic import Extra, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.utils import get_from_dict_or_env
TIMEOUT = 60
[docs]@dataclasses.dataclass
class AviaryBackend:
    backend_url: str
    bearer: str
    def __post_init__(self) -> None:
        self.header = {"Authorization": self.bearer}
[docs]    @classmethod
    def from_env(cls) -> "AviaryBackend":
        aviary_url = os.getenv("AVIARY_URL")
        assert aviary_url, "AVIARY_URL must be set"
        aviary_token = os.getenv("AVIARY_TOKEN", "")
        bearer = f"Bearer {aviary_token}" if aviary_token else ""
        aviary_url += "/" if not aviary_url.endswith("/") else ""
        return cls(aviary_url, bearer)
[docs]def get_models() -> List[str]:
    """List available models"""
    backend = AviaryBackend.from_env()
    request_url = backend.backend_url + "-/routes"
    response = requests.get(request_url, headers=backend.header, timeout=TIMEOUT)
    try:
        result = response.json()
    except requests.JSONDecodeError as e:
        raise RuntimeError(
            f"Error decoding JSON from {request_url}. Text response: {response.text}"
        ) from e
    result = sorted(


) from e
    result = sorted(
        [k.lstrip("/").replace("--", "/") for k in result.keys() if "--" in k]
    )
    return result
[docs]def get_completions(
    model: str,
    prompt: str,
    use_prompt_format: bool = True,
    version: str = "",
) -> Dict[str, Union[str, float, int]]:
    """Get completions from Aviary models."""
    backend = AviaryBackend.from_env()
    url = backend.backend_url + model.replace("/", "--") + "/" + version + "query"
    response = requests.post(
        url,
        headers=backend.header,
        json={"prompt": prompt, "use_prompt_format": use_prompt_format},
        timeout=TIMEOUT,
    )
    try:
        return response.json()
    except requests.JSONDecodeError as e:
        raise RuntimeError(
            f"Error decoding JSON from {url}. Text response: {response.text}"
        ) from e
[docs]class Aviary(LLM):
    """Aviary hosted models.
    Aviary is a backend for hosted models. You can
    find out more about aviary at
    http://github.com/ray-project/aviary
    To get a list of the models supported on an
    aviary, follow the instructions on the website to
    install the aviary CLI and then use:
    `aviary models`
    AVIARY_URL and AVIARY_TOKEN environment variables must be set.
    Example:
        .. code-block:: python
            from langchain.llms import Aviary
            os.environ["AVIARY_URL"] = "<URL>"
            os.environ["AVIARY_TOKEN"] = "<TOKEN>"


os.environ["AVIARY_TOKEN"] = "<TOKEN>"
            light = Aviary(model='amazon/LightGPT')
            output = light('How do you make fried rice?')
    """
    model: str = "amazon/LightGPT"
    aviary_url: Optional[str] = None
    aviary_token: Optional[str] = None
    # If True the prompt template for the model will be ignored.
    use_prompt_format: bool = True
    # API version to use for Aviary
    version: Optional[str] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        aviary_url = get_from_dict_or_env(values, "aviary_url", "AVIARY_URL")
        aviary_token = get_from_dict_or_env(values, "aviary_token", "AVIARY_TOKEN")
        # Set env viarables for aviary sdk
        os.environ["AVIARY_URL"] = aviary_url
        os.environ["AVIARY_TOKEN"] = aviary_token
        try:
            aviary_models = get_models()
        except requests.exceptions.RequestException as e:
            raise ValueError(e)
        model = values.get("model")
        if model and model not in aviary_models:
            raise ValueError(f"{aviary_url} does not support model {values['model']}.")
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_name": self.model,
            "aviary_url": self.aviary_url,


"aviary_url": self.aviary_url,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return f"aviary-{self.model.replace('/', '-')}"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Aviary
        Args:
            prompt: The prompt to pass into the model.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = aviary("Tell me a joke.")
        """
        kwargs = {"use_prompt_format": self.use_prompt_format}
        if self.version:
            kwargs["version"] = self.version
        output = get_completions(
            model=self.model,
            prompt=prompt,
            **kwargs,
        )
        text = cast(str, output["generated_text"])
        if stop:
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.huggingface_pipeline
import importlib.util
import logging
from typing import Any, List, Mapping, Optional
from pydantic import Extra
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
DEFAULT_MODEL_ID = "gpt2"
DEFAULT_TASK = "text-generation"
VALID_TASKS = ("text2text-generation", "text-generation", "summarization")
logger = logging.getLogger(__name__)
[docs]class HuggingFacePipeline(LLM):
    """HuggingFace Pipeline API.
    To use, you should have the ``transformers`` python package installed.
    Only supports `text-generation`, `text2text-generation` and `summarization` for now.
    Example using from_model_id:
        .. code-block:: python
            from langchain.llms import HuggingFacePipeline
            hf = HuggingFacePipeline.from_model_id(
                model_id="gpt2",
                task="text-generation",
                pipeline_kwargs={"max_new_tokens": 10},
            )
    Example passing pipeline in directly:
        .. code-block:: python
            from langchain.llms import HuggingFacePipeline
            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
            model_id = "gpt2"
            tokenizer = AutoTokenizer.from_pretrained(model_id)
            model = AutoModelForCausalLM.from_pretrained(model_id)
            pipe = pipeline(
                "text-generation", model=model, tokenizer=tokenizer, max_new_tokens=10
            )
            hf = HuggingFacePipeline(pipeline=pipe)
    """
    pipeline: Any  #: :meta private:
    model_id: str = DEFAULT_MODEL_ID


model_id: str = DEFAULT_MODEL_ID
    """Model name to use."""
    model_kwargs: Optional[dict] = None
    """Key word arguments passed to the model."""
    pipeline_kwargs: Optional[dict] = None
    """Key word arguments passed to the pipeline."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
[docs]    @classmethod
    def from_model_id(
        cls,
        model_id: str,
        task: str,
        device: int = -1,
        model_kwargs: Optional[dict] = None,
        pipeline_kwargs: Optional[dict] = None,
        **kwargs: Any,
    ) -> LLM:
        """Construct the pipeline object from model_id and task."""
        try:
            from transformers import (
                AutoModelForCausalLM,
                AutoModelForSeq2SeqLM,
                AutoTokenizer,
            )
            from transformers import pipeline as hf_pipeline
        except ImportError:
            raise ValueError(
                "Could not import transformers python package. "
                "Please install it with `pip install transformers`."
            )
        _model_kwargs = model_kwargs or {}
        tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs)
        try:
            if task == "text-generation":
                model = AutoModelForCausalLM.from_pretrained(model_id, **_model_kwargs)
            elif task in ("text2text-generation", "summarization"):
                model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **_model_kwargs)
            else:
                raise ValueError(
                    f"Got invalid task {task}, "
                    f"currently only {VALID_TASKS} are supported"
                )


f"currently only {VALID_TASKS} are supported"
                )
        except ImportError as e:
            raise ValueError(
                f"Could not load the {task} model due to missing dependencies."
            ) from e
        if importlib.util.find_spec("torch") is not None:
            import torch
            cuda_device_count = torch.cuda.device_count()
            if device < -1 or (device >= cuda_device_count):
                raise ValueError(
                    f"Got device=={device}, "
                    f"device is required to be within [-1, {cuda_device_count})"
                )
            if device < 0 and cuda_device_count > 0:
                logger.warning(
                    "Device has %d GPUs available. "
                    "Provide device={deviceId} to `from_model_id` to use available"
                    "GPUs for execution. deviceId is -1 (default) for CPU and "
                    "can be a positive integer associated with CUDA device id.",
                    cuda_device_count,
                )
        if "trust_remote_code" in _model_kwargs:
            _model_kwargs = {
                k: v for k, v in _model_kwargs.items() if k != "trust_remote_code"
            }
        _pipeline_kwargs = pipeline_kwargs or {}
        pipeline = hf_pipeline(
            task=task,
            model=model,
            tokenizer=tokenizer,
            device=device,
            model_kwargs=_model_kwargs,
            **_pipeline_kwargs,
        )
        if pipeline.task not in VALID_TASKS:
            raise ValueError(
                f"Got invalid task {pipeline.task}, "
                f"currently only {VALID_TASKS} are supported"
            )
        return cls(
            pipeline=pipeline,
            model_id=model_id,
            model_kwargs=_model_kwargs,


model_id=model_id,
            model_kwargs=_model_kwargs,
            pipeline_kwargs=_pipeline_kwargs,
            **kwargs,
        )
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_id": self.model_id,
            "model_kwargs": self.model_kwargs,
            "pipeline_kwargs": self.pipeline_kwargs,
        }
    @property
    def _llm_type(self) -> str:
        return "huggingface_pipeline"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        response = self.pipeline(prompt)
        if self.pipeline.task == "text-generation":
            # Text generation return includes the starter text.
            text = response[0]["generated_text"][len(prompt) :]
        elif self.pipeline.task == "text2text-generation":
            text = response[0]["generated_text"]
        elif self.pipeline.task == "summarization":
            text = response[0]["summary_text"]
        else:
            raise ValueError(
                f"Got invalid task {self.pipeline.task}, "
                f"currently only {VALID_TASKS} are supported"
            )
        if stop:
            # This is a bit hacky, but I can't figure out a better way to enforce
            # stop tokens when making calls to huggingface_hub.
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.writer
from typing import Any, Dict, List, Mapping, Optional
import requests
from pydantic import Extra, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.utils import get_from_dict_or_env
[docs]class Writer(LLM):
    """Writer large language models.
    To use, you should have the environment variable ``WRITER_API_KEY`` and
    ``WRITER_ORG_ID`` set with your API key and organization ID respectively.
    Example:
        .. code-block:: python
            from langchain import Writer
            writer = Writer(model_id="palmyra-base")
    """
    writer_org_id: Optional[str] = None
    """Writer organization ID."""
    model_id: str = "palmyra-instruct"
    """Model name to use."""
    min_tokens: Optional[int] = None
    """Minimum number of tokens to generate."""
    max_tokens: Optional[int] = None
    """Maximum number of tokens to generate."""
    temperature: Optional[float] = None
    """What sampling temperature to use."""
    top_p: Optional[float] = None
    """Total probability mass of tokens to consider at each step."""
    stop: Optional[List[str]] = None
    """Sequences when completion generation will stop."""
    presence_penalty: Optional[float] = None
    """Penalizes repeated tokens regardless of frequency."""
    repetition_penalty: Optional[float] = None
    """Penalizes repeated tokens according to frequency."""
    best_of: Optional[int] = None
    """Generates this many completions server-side and returns the "best"."""
    logprobs: bool = False
    """Whether to return log probabilities."""


logprobs: bool = False
    """Whether to return log probabilities."""
    n: Optional[int] = None
    """How many completions to generate."""
    writer_api_key: Optional[str] = None
    """Writer API key."""
    base_url: Optional[str] = None
    """Base url to use, if None decides based on model name."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and organization id exist in environment."""
        writer_api_key = get_from_dict_or_env(
            values, "writer_api_key", "WRITER_API_KEY"
        )
        values["writer_api_key"] = writer_api_key
        writer_org_id = get_from_dict_or_env(values, "writer_org_id", "WRITER_ORG_ID")
        values["writer_org_id"] = writer_org_id
        return values
    @property
    def _default_params(self) -> Mapping[str, Any]:
        """Get the default parameters for calling Writer API."""
        return {
            "minTokens": self.min_tokens,
            "maxTokens": self.max_tokens,
            "temperature": self.temperature,
            "topP": self.top_p,
            "stop": self.stop,
            "presencePenalty": self.presence_penalty,
            "repetitionPenalty": self.repetition_penalty,
            "bestOf": self.best_of,
            "logprobs": self.logprobs,
            "n": self.n,
        }
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {


"""Get the identifying parameters."""
        return {
            **{"model_id": self.model_id, "writer_org_id": self.writer_org_id},
            **self._default_params,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "writer"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Writer's completions endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = Writer("Tell me a joke.")
        """
        if self.base_url is not None:
            base_url = self.base_url
        else:
            base_url = (
                "https://enterprise-api.writer.com/llm"
                f"/organization/{self.writer_org_id}"
                f"/model/{self.model_id}/completions"
            )
        params = {**self._default_params, **kwargs}
        response = requests.post(
            url=base_url,
            headers={
                "Authorization": f"{self.writer_api_key}",
                "Content-Type": "application/json",
                "Accept": "application/json",
            },
            json={"prompt": prompt, **params},
        )
        text = response.text
        if stop is not None:
            # I believe this is required since the stop tokens
            # are not enforced by the model parameters


# are not enforced by the model parameters
            text = enforce_stop_tokens(text, stop)
        return text


Source code for langchain.llms.beam
import base64
import json
import logging
import subprocess
import textwrap
import time
from typing import Any, Dict, List, Mapping, Optional
import requests
from pydantic import Extra, Field, root_validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
DEFAULT_NUM_TRIES = 10
DEFAULT_SLEEP_TIME = 4
[docs]class Beam(LLM):
    """Beam API for gpt2 large language model.
    To use, you should have the ``beam-sdk`` python package installed,
    and the environment variable ``BEAM_CLIENT_ID`` set with your client id
    and ``BEAM_CLIENT_SECRET`` set with your client secret. Information on how
    to get this is available here: https://docs.beam.cloud/account/api-keys.
    The wrapper can then be called as follows, where the name, cpu, memory, gpu,
    python version, and python packages can be updated accordingly. Once deployed,
    the instance can be called.
    Example:
        .. code-block:: python
            llm = Beam(model_name="gpt2",
                name="langchain-gpt2",
                cpu=8,
                memory="32Gi",
                gpu="A10G",
                python_version="python3.8",
                python_packages=[
                    "diffusers[torch]>=0.10",
                    "transformers",
                    "torch",
                    "pillow",
                    "accelerate",
                    "safetensors",
                    "xformers",],
                max_length=50)
            llm._deploy()
            call_result = llm._call(input)


llm._deploy()
            call_result = llm._call(input)
    """
    model_name: str = ""
    name: str = ""
    cpu: str = ""
    memory: str = ""
    gpu: str = ""
    python_version: str = ""
    python_packages: List[str] = []
    max_length: str = ""
    url: str = ""
    """model endpoint to use"""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not
    explicitly specified."""
    beam_client_id: str = ""
    beam_client_secret: str = ""
    app_id: Optional[str] = None
    class Config:
        """Configuration for this pydantic config."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""{field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""


"""Validate that api key and python package exists in environment."""
        beam_client_id = get_from_dict_or_env(
            values, "beam_client_id", "BEAM_CLIENT_ID"
        )
        beam_client_secret = get_from_dict_or_env(
            values, "beam_client_secret", "BEAM_CLIENT_SECRET"
        )
        values["beam_client_id"] = beam_client_id
        values["beam_client_secret"] = beam_client_secret
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_name": self.model_name,
            "name": self.name,
            "cpu": self.cpu,
            "memory": self.memory,
            "gpu": self.gpu,
            "python_version": self.python_version,
            "python_packages": self.python_packages,
            "max_length": self.max_length,
            "model_kwargs": self.model_kwargs,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "beam"
[docs]    def app_creation(self) -> None:
        """Creates a Python file which will contain your Beam app definition."""
        script = textwrap.dedent(
            """\
        import beam
        # The environment your code will run on
        app = beam.App(
            name="{name}",
            cpu={cpu},
            memory="{memory}",
            gpu="{gpu}",
            python_version="{python_version}",
            python_packages={python_packages},
        )
        app.Trigger.RestAPI(
            inputs={{"prompt": beam.Types.String(), "max_length": beam.Types.String()}},
            outputs={{"text": beam.Types.String()}},


outputs={{"text": beam.Types.String()}},
            handler="run.py:beam_langchain",
        )
        """
        )
        script_name = "app.py"
        with open(script_name, "w") as file:
            file.write(
                script.format(
                    name=self.name,
                    cpu=self.cpu,
                    memory=self.memory,
                    gpu=self.gpu,
                    python_version=self.python_version,
                    python_packages=self.python_packages,
                )
            )
[docs]    def run_creation(self) -> None:
        """Creates a Python file which will be deployed on beam."""
        script = textwrap.dedent(
            """
        import os
        import transformers
        from transformers import GPT2LMHeadModel, GPT2Tokenizer
        model_name = "{model_name}"
        def beam_langchain(**inputs):
            prompt = inputs["prompt"]
            length = inputs["max_length"]
            tokenizer = GPT2Tokenizer.from_pretrained(model_name)
            model = GPT2LMHeadModel.from_pretrained(model_name)
            encodedPrompt = tokenizer.encode(prompt, return_tensors='pt')
            outputs = model.generate(encodedPrompt, max_length=int(length),
              do_sample=True, pad_token_id=tokenizer.eos_token_id)
            output = tokenizer.decode(outputs[0], skip_special_tokens=True)
            print(output)
            return {{"text": output}}
        """
        )
        script_name = "run.py"
        with open(script_name, "w") as file:
            file.write(script.format(model_name=self.model_name))
    def _deploy(self) -> str:
        """Call to Beam."""
        try:
            import beam  # type: ignore
            if beam.__path__ == "":
                raise ImportError


if beam.__path__ == "":
                raise ImportError
        except ImportError:
            raise ImportError(
                "Could not import beam python package. "
                "Please install it with `curl "
                "https://raw.githubusercontent.com/slai-labs"
                "/get-beam/main/get-beam.sh -sSfL | sh`."
            )
        self.app_creation()
        self.run_creation()
        process = subprocess.run(
            "beam deploy app.py", shell=True, capture_output=True, text=True
        )
        if process.returncode == 0:
            output = process.stdout
            logger.info(output)
            lines = output.split("\n")
            for line in lines:
                if line.startswith(" i  Send requests to: https://apps.beam.cloud/"):
                    self.app_id = line.split("/")[-1]
                    self.url = line.split(":")[1].strip()
                    return self.app_id
            raise ValueError(
                f"""Failed to retrieve the appID from the deployment output.
                Deployment output: {output}"""
            )
        else:
            raise ValueError(f"Deployment failed. Error: {process.stderr}")
    @property
    def authorization(self) -> str:
        if self.beam_client_id:
            credential_str = self.beam_client_id + ":" + self.beam_client_secret
        else:
            credential_str = self.beam_client_secret
        return base64.b64encode(credential_str.encode()).decode()
    def _call(
        self,
        prompt: str,
        stop: Optional[list] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call to Beam."""


**kwargs: Any,
    ) -> str:
        """Call to Beam."""
        url = "https://apps.beam.cloud/" + self.app_id if self.app_id else self.url
        payload = {"prompt": prompt, "max_length": self.max_length}
        payload.update(kwargs)
        headers = {
            "Accept": "*/*",
            "Accept-Encoding": "gzip, deflate",
            "Authorization": "Basic " + self.authorization,
            "Connection": "keep-alive",
            "Content-Type": "application/json",
        }
        for _ in range(DEFAULT_NUM_TRIES):
            request = requests.post(url, headers=headers, data=json.dumps(payload))
            if request.status_code == 200:
                return request.json()["text"]
            time.sleep(DEFAULT_SLEEP_TIME)
        logger.warning("Unable to successfully call model.")
        return ""


Source code for langchain.llms.cohere
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, List, Optional
from pydantic import Extra, root_validator
from tenacity import (
    before_sleep_log,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential,
)
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
def _create_retry_decorator(llm: Cohere) -> Callable[[Any], Any]:
    import cohere
    min_seconds = 4
    max_seconds = 10
    # Wait 2^x * 1 second between each retry starting with
    # 4 seconds, then up to 10 seconds, then 10 seconds afterwards
    return retry(
        reraise=True,
        stop=stop_after_attempt(llm.max_retries),
        wait=wait_exponential(multiplier=1, min=min_seconds, max=max_seconds),
        retry=(retry_if_exception_type(cohere.error.CohereError)),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
[docs]def completion_with_retry(llm: Cohere, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm)
    @retry_decorator
    def _completion_with_retry(**kwargs: Any) -> Any:
        return llm.client.generate(**kwargs)
    return _completion_with_retry(**kwargs)


return llm.client.generate(**kwargs)
    return _completion_with_retry(**kwargs)
[docs]def acompletion_with_retry(llm: Cohere, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm)
    @retry_decorator
    async def _completion_with_retry(**kwargs: Any) -> Any:
        return await llm.async_client.generate(**kwargs)
    return _completion_with_retry(**kwargs)
[docs]class Cohere(LLM):
    """Cohere large language models.
    To use, you should have the ``cohere`` python package installed, and the
    environment variable ``COHERE_API_KEY`` set with your API key, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain.llms import Cohere
            cohere = Cohere(model="gptd-instruct-tft", cohere_api_key="my-api-key")
    """
    client: Any  #: :meta private:
    async_client: Any  #: :meta private:
    model: Optional[str] = None
    """Model name to use."""
    max_tokens: int = 256
    """Denotes the number of tokens to predict per generation."""
    temperature: float = 0.75
    """A non-negative float that tunes the degree of randomness in generation."""
    k: int = 0
    """Number of most likely tokens to consider at each step."""
    p: int = 1
    """Total probability mass of tokens to consider at each step."""
    frequency_penalty: float = 0.0
    """Penalizes repeated tokens according to frequency. Between 0 and 1."""


"""Penalizes repeated tokens according to frequency. Between 0 and 1."""
    presence_penalty: float = 0.0
    """Penalizes repeated tokens. Between 0 and 1."""
    truncate: Optional[str] = None
    """Specify how the client handles inputs longer than the maximum token
    length: Truncate from START, END or NONE"""
    max_retries: int = 10
    """Maximum number of retries to make when generating."""
    cohere_api_key: Optional[str] = None
    stop: Optional[List[str]] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        cohere_api_key = get_from_dict_or_env(
            values, "cohere_api_key", "COHERE_API_KEY"
        )
        try:
            import cohere
            values["client"] = cohere.Client(cohere_api_key)
            values["async_client"] = cohere.AsyncClient(cohere_api_key)
        except ImportError:
            raise ImportError(
                "Could not import cohere python package. "
                "Please install it with `pip install cohere`."
            )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling Cohere API."""
        return {
            "max_tokens": self.max_tokens,
            "temperature": self.temperature,
            "k": self.k,
            "p": self.p,
            "frequency_penalty": self.frequency_penalty,
            "presence_penalty": self.presence_penalty,
            "truncate": self.truncate,


"presence_penalty": self.presence_penalty,
            "truncate": self.truncate,
        }
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {**{"model": self.model}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "cohere"
    def _invocation_params(self, stop: Optional[List[str]], **kwargs: Any) -> dict:
        params = self._default_params
        if self.stop is not None and stop is not None:
            raise ValueError("`stop` found in both the input and default params.")
        elif self.stop is not None:
            params["stop_sequences"] = self.stop
        else:
            params["stop_sequences"] = stop
        return {**params, **kwargs}
    def _process_response(self, response: Any, stop: Optional[List[str]]) -> str:
        text = response.generations[0].text
        # If stop tokens are provided, Cohere's endpoint returns them.
        # In order to make this consistent with other endpoints, we strip them.
        if stop:
            text = enforce_stop_tokens(text, stop)
        return text
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Cohere's generate endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.


Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = cohere("Tell me a joke.")
        """
        params = self._invocation_params(stop, **kwargs)
        response = completion_with_retry(
            self, model=self.model, prompt=prompt, **params
        )
        _stop = params.get("stop_sequences")
        return self._process_response(response, _stop)
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Async call out to Cohere's generate endpoint.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = await cohere("Tell me a joke.")
        """
        params = self._invocation_params(stop, **kwargs)
        response = await acompletion_with_retry(
            self, model=self.model, prompt=prompt, **params
        )
        _stop = params.get("stop_sequences")
        return self._process_response(response, _stop)


Source code for langchain.llms.vertexai
from __future__ import annotations
import asyncio
from concurrent.futures import Executor, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional
from pydantic import BaseModel, root_validator
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM, create_base_retry_decorator
from langchain.llms.utils import enforce_stop_tokens
from langchain.utilities.vertexai import (
    init_vertexai,
    raise_vertex_import_error,
)
if TYPE_CHECKING:
    from vertexai.language_models._language_models import _LanguageModel
[docs]def is_codey_model(model_name: str) -> bool:
    """Returns True if the model name is a Codey model.
    Args:
        model_name: The model name to check.
    Returns: True if the model name is a Codey model.
    """
    return "code" in model_name
def _create_retry_decorator(llm: VertexAI) -> Callable[[Any], Any]:
    import google.api_core
    errors = [
        google.api_core.exceptions.ResourceExhausted,
        google.api_core.exceptions.ServiceUnavailable,
        google.api_core.exceptions.Aborted,
        google.api_core.exceptions.DeadlineExceeded,
    ]
    decorator = create_base_retry_decorator(
        error_types=errors, max_retries=llm.max_retries  # type: ignore
    )
    return decorator
[docs]def completion_with_retry(llm: VertexAI, *args: Any, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator(llm)
    @retry_decorator


retry_decorator = _create_retry_decorator(llm)
    @retry_decorator
    def _completion_with_retry(*args: Any, **kwargs: Any) -> Any:
        return llm.client.predict(*args, **kwargs)
    return _completion_with_retry(*args, **kwargs)
class _VertexAICommon(BaseModel):
    client: "_LanguageModel" = None  #: :meta private:
    model_name: str
    "Model name to use."
    temperature: float = 0.0
    "Sampling temperature, it controls the degree of randomness in token selection."
    max_output_tokens: int = 128
    "Token limit determines the maximum amount of text output from one prompt."
    top_p: float = 0.95
    "Tokens are selected from most probable to least until the sum of their "
    "probabilities equals the top-p value. Top-p is ignored for Codey models."
    top_k: int = 40
    "How the model selects tokens for output, the next token is selected from "
    "among the top-k most probable tokens. Top-k is ignored for Codey models."
    stop: Optional[List[str]] = None
    "Optional list of stop words to use when generating."
    project: Optional[str] = None
    "The default GCP project to use when making Vertex API calls."
    location: str = "us-central1"
    "The default location to use when making API calls."
    credentials: Any = None
    "The default custom credentials (google.auth.credentials.Credentials) to use "
    "when making API calls. If not provided, credentials will be ascertained from "
    "the environment."
    request_parallelism: int = 5
    "The amount of parallelism allowed for requests issued to VertexAI models. "


"The amount of parallelism allowed for requests issued to VertexAI models. "
    "Default is 5."
    max_retries: int = 6
    """The maximum number of retries to make when generating."""
    task_executor: ClassVar[Optional[Executor]] = None
    @property
    def is_codey_model(self) -> bool:
        return is_codey_model(self.model_name)
    @property
    def _default_params(self) -> Dict[str, Any]:
        if self.is_codey_model:
            return {
                "temperature": self.temperature,
                "max_output_tokens": self.max_output_tokens,
            }
        else:
            return {
                "temperature": self.temperature,
                "max_output_tokens": self.max_output_tokens,
                "top_k": self.top_k,
                "top_p": self.top_p,
            }
    def _predict(
        self, prompt: str, stop: Optional[List[str]] = None, **kwargs: Any
    ) -> str:
        params = {**self._default_params, **kwargs}
        res = completion_with_retry(self, prompt, **params)  # type: ignore
        return self._enforce_stop_words(res.text, stop)
    def _enforce_stop_words(self, text: str, stop: Optional[List[str]] = None) -> str:
        if stop is None and self.stop is not None:
            stop = self.stop
        if stop:
            return enforce_stop_tokens(text, stop)
        return text
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {**{"model_name": self.model_name}, **self._default_params}
    @property


@property
    def _llm_type(self) -> str:
        return "vertexai"
    @classmethod
    def _get_task_executor(cls, request_parallelism: int = 5) -> Executor:
        if cls.task_executor is None:
            cls.task_executor = ThreadPoolExecutor(max_workers=request_parallelism)
        return cls.task_executor
    @classmethod
    def _try_init_vertexai(cls, values: Dict) -> None:
        allowed_params = ["project", "location", "credentials"]
        params = {k: v for k, v in values.items() if k in allowed_params}
        init_vertexai(**params)
        return None
[docs]class VertexAI(_VertexAICommon, LLM):
    """Google Vertex AI large language models."""
    model_name: str = "text-bison"
    "The name of the Vertex AI large language model."
    tuned_model_name: Optional[str] = None
    "The name of a tuned model. If provided, model_name is ignored."
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that the python package exists in environment."""
        cls._try_init_vertexai(values)
        tuned_model_name = values.get("tuned_model_name")
        model_name = values["model_name"]
        try:
            if tuned_model_name or not is_codey_model(model_name):
                from vertexai.preview.language_models import TextGenerationModel
                if tuned_model_name:
                    values["client"] = TextGenerationModel.get_tuned_model(
                        tuned_model_name
                    )
                else:
                    values["client"] = TextGenerationModel.from_pretrained(model_name)
            else:
                from vertexai.preview.language_models import CodeGenerationModel


else:
                from vertexai.preview.language_models import CodeGenerationModel
                values["client"] = CodeGenerationModel.from_pretrained(model_name)
        except ImportError:
            raise_vertex_import_error()
        return values
    async def _acall(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call Vertex model to get predictions based on the prompt.
        Args:
            prompt: The prompt to pass into the model.
            stop: A list of stop words (optional).
            run_manager: A callback manager for async interaction with LLMs.
        Returns:
            The string generated by the model.
        """
        return await asyncio.wrap_future(
            self._get_task_executor().submit(self._predict, prompt, stop)
        )
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call Vertex model to get predictions based on the prompt.
        Args:
            prompt: The prompt to pass into the model.
            stop: A list of stop words (optional).
            run_manager: A Callbackmanager for LLM run, optional.
        Returns:
            The string generated by the model.
        """
        return self._predict(prompt, stop, **kwargs)


Source code for langchain.llms.minimax
"""Wrapper around Minimax APIs."""
from __future__ import annotations
import logging
from typing import (
    Any,
    Dict,
    List,
    Optional,
)
import requests
from pydantic import BaseModel, Extra, Field, PrivateAttr, root_validator
from langchain.callbacks.manager import (
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
class _MinimaxEndpointClient(BaseModel):
    """An API client that talks to a Minimax llm endpoint."""
    host: str
    group_id: str
    api_key: str
    api_url: str
    @root_validator(pre=True)
    def set_api_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        if "api_url" not in values:
            host = values["host"]
            group_id = values["group_id"]
            api_url = f"{host}/v1/text/chatcompletion?GroupId={group_id}"
            values["api_url"] = api_url
        return values
    def post(self, request: Any) -> Any:
        headers = {"Authorization": f"Bearer {self.api_key}"}
        response = requests.post(self.api_url, headers=headers, json=request)
        # TODO: error handling and automatic retries
        if not response.ok:
            raise ValueError(f"HTTP {response.status_code} error: {response.text}")
        if response.json()["base_resp"]["status_code"] > 0:
            raise ValueError(
                f"API {response.json()['base_resp']['status_code']}"
                f" error: {response.json()['base_resp']['status_msg']}"
            )


f" error: {response.json()['base_resp']['status_msg']}"
            )
        return response.json()["reply"]
[docs]class Minimax(LLM):
    """Wrapper around Minimax large language models.
    To use, you should have the environment variable
    ``MINIMAX_API_KEY`` and ``MINIMAX_GROUP_ID`` set with your API key,
    or pass them as a named parameter to the constructor.
    Example:
     .. code-block:: python
         from langchain.llms.minimax import Minimax
         minimax = Minimax(model="<model_name>", minimax_api_key="my-api-key",
          minimax_group_id="my-group-id")
    """
    _client: _MinimaxEndpointClient = PrivateAttr()
    model: str = "abab5.5-chat"
    """Model name to use."""
    max_tokens: int = 256
    """Denotes the number of tokens to predict per generation."""
    temperature: float = 0.7
    """A non-negative float that tunes the degree of randomness in generation."""
    top_p: float = 0.95
    """Total probability mass of tokens to consider at each step."""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not explicitly specified."""
    minimax_api_host: Optional[str] = None
    minimax_group_id: Optional[str] = None
    minimax_api_key: Optional[str] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""


"""Validate that api key and python package exists in environment."""
        values["minimax_api_key"] = get_from_dict_or_env(
            values, "minimax_api_key", "MINIMAX_API_KEY"
        )
        values["minimax_group_id"] = get_from_dict_or_env(
            values, "minimax_group_id", "MINIMAX_GROUP_ID"
        )
        # Get custom api url from environment.
        values["minimax_api_host"] = get_from_dict_or_env(
            values,
            "minimax_api_host",
            "MINIMAX_API_HOST",
            default="https://api.minimax.chat",
        )
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling OpenAI API."""
        return {
            "model": self.model,
            "tokens_to_generate": self.max_tokens,
            "temperature": self.temperature,
            "top_p": self.top_p,
            **self.model_kwargs,
        }
    @property
    def _identifying_params(self) -> Dict[str, Any]:
        """Get the identifying parameters."""
        return {**{"model": self.model}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "minimax"
    def __init__(self, **data: Any):
        super().__init__(**data)
        self._client = _MinimaxEndpointClient(
            host=self.minimax_api_host,
            api_key=self.minimax_api_key,
            group_id=self.minimax_group_id,
        )
    def _call(
        self,
        prompt: str,


)
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        r"""Call out to Minimax's completion endpoint to chat
        Args:
            prompt: The prompt to pass into the model.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = minimax("Tell me a joke.")
        """
        request = self._default_params
        request["messages"] = [{"sender_type": "USER", "text": prompt}]
        request.update(kwargs)
        response = self._client.post(request)
        return response


Source code for langchain.llms.self_hosted_hugging_face
import importlib.util
import logging
from typing import Any, Callable, List, Mapping, Optional
from pydantic import Extra
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.self_hosted import SelfHostedPipeline
from langchain.llms.utils import enforce_stop_tokens
DEFAULT_MODEL_ID = "gpt2"
DEFAULT_TASK = "text-generation"
VALID_TASKS = ("text2text-generation", "text-generation", "summarization")
logger = logging.getLogger(__name__)
def _generate_text(
    pipeline: Any,
    prompt: str,
    *args: Any,
    stop: Optional[List[str]] = None,
    **kwargs: Any,
) -> str:
    """Inference function to send to the remote hardware.
    Accepts a Hugging Face pipeline (or more likely,
    a key pointing to such a pipeline on the cluster's object store)
    and returns generated text.
    """
    response = pipeline(prompt, *args, **kwargs)
    if pipeline.task == "text-generation":
        # Text generation return includes the starter text.
        text = response[0]["generated_text"][len(prompt) :]
    elif pipeline.task == "text2text-generation":
        text = response[0]["generated_text"]
    elif pipeline.task == "summarization":
        text = response[0]["summary_text"]
    else:
        raise ValueError(
            f"Got invalid task {pipeline.task}, "
            f"currently only {VALID_TASKS} are supported"
        )
    if stop is not None:
        text = enforce_stop_tokens(text, stop)
    return text
def _load_transformer(
    model_id: str = DEFAULT_MODEL_ID,


def _load_transformer(
    model_id: str = DEFAULT_MODEL_ID,
    task: str = DEFAULT_TASK,
    device: int = 0,
    model_kwargs: Optional[dict] = None,
) -> Any:
    """Inference function to send to the remote hardware.
    Accepts a huggingface model_id and returns a pipeline for the task.
    """
    from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer
    from transformers import pipeline as hf_pipeline
    _model_kwargs = model_kwargs or {}
    tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs)
    try:
        if task == "text-generation":
            model = AutoModelForCausalLM.from_pretrained(model_id, **_model_kwargs)
        elif task in ("text2text-generation", "summarization"):
            model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **_model_kwargs)
        else:
            raise ValueError(
                f"Got invalid task {task}, "
                f"currently only {VALID_TASKS} are supported"
            )
    except ImportError as e:
        raise ValueError(
            f"Could not load the {task} model due to missing dependencies."
        ) from e
    if importlib.util.find_spec("torch") is not None:
        import torch
        cuda_device_count = torch.cuda.device_count()
        if device < -1 or (device >= cuda_device_count):
            raise ValueError(
                f"Got device=={device}, "
                f"device is required to be within [-1, {cuda_device_count})"
            )
        if device < 0 and cuda_device_count > 0:
            logger.warning(


if device < 0 and cuda_device_count > 0:
            logger.warning(
                "Device has %d GPUs available. "
                "Provide device={deviceId} to `from_model_id` to use available"
                "GPUs for execution. deviceId is -1 for CPU and "
                "can be a positive integer associated with CUDA device id.",
                cuda_device_count,
            )
    pipeline = hf_pipeline(
        task=task,
        model=model,
        tokenizer=tokenizer,
        device=device,
        model_kwargs=_model_kwargs,
    )
    if pipeline.task not in VALID_TASKS:
        raise ValueError(
            f"Got invalid task {pipeline.task}, "
            f"currently only {VALID_TASKS} are supported"
        )
    return pipeline
[docs]class SelfHostedHuggingFaceLLM(SelfHostedPipeline):
    """HuggingFace Pipeline API to run on self-hosted remote hardware.
    Supported hardware includes auto-launched instances on AWS, GCP, Azure,
    and Lambda, as well as servers specified
    by IP address and SSH credentials (such as on-prem, or another cloud
    like Paperspace, Coreweave, etc.).
    To use, you should have the ``runhouse`` python package installed.
    Only supports `text-generation`, `text2text-generation` and `summarization` for now.
    Example using from_model_id:
        .. code-block:: python
            from langchain.llms import SelfHostedHuggingFaceLLM
            import runhouse as rh
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            hf = SelfHostedHuggingFaceLLM(
                model_id="google/flan-t5-large", task="text2text-generation",


model_id="google/flan-t5-large", task="text2text-generation",
                hardware=gpu
            )
    Example passing fn that generates a pipeline (bc the pipeline is not serializable):
        .. code-block:: python
            from langchain.llms import SelfHostedHuggingFaceLLM
            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
            import runhouse as rh
            def get_pipeline():
                model_id = "gpt2"
                tokenizer = AutoTokenizer.from_pretrained(model_id)
                model = AutoModelForCausalLM.from_pretrained(model_id)
                pipe = pipeline(
                    "text-generation", model=model, tokenizer=tokenizer
                )
                return pipe
            hf = SelfHostedHuggingFaceLLM(
                model_load_fn=get_pipeline, model_id="gpt2", hardware=gpu)
    """
    model_id: str = DEFAULT_MODEL_ID
    """Hugging Face model_id to load the model."""
    task: str = DEFAULT_TASK
    """Hugging Face task ("text-generation", "text2text-generation" or
    "summarization")."""
    device: int = 0
    """Device to use for inference. -1 for CPU, 0 for GPU, 1 for second GPU, etc."""
    model_kwargs: Optional[dict] = None
    """Key word arguments to pass to the model."""
    hardware: Any
    """Remote hardware to send the inference function to."""
    model_reqs: List[str] = ["./", "transformers", "torch"]
    """Requirements to install on hardware to inference the model."""
    model_load_fn: Callable = _load_transformer
    """Function to load the model remotely on the server."""


"""Function to load the model remotely on the server."""
    inference_fn: Callable = _generate_text  #: :meta private:
    """Inference function to send to the remote hardware."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    def __init__(self, **kwargs: Any):
        """Construct the pipeline remotely using an auxiliary function.
        The load function needs to be importable to be imported
        and run on the server, i.e. in a module and not a REPL or closure.
        Then, initialize the remote inference function.
        """
        load_fn_kwargs = {
            "model_id": kwargs.get("model_id", DEFAULT_MODEL_ID),
            "task": kwargs.get("task", DEFAULT_TASK),
            "device": kwargs.get("device", 0),
            "model_kwargs": kwargs.get("model_kwargs", None),
        }
        super().__init__(load_fn_kwargs=load_fn_kwargs, **kwargs)
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            **{"model_id": self.model_id},
            **{"model_kwargs": self.model_kwargs},
        }
    @property
    def _llm_type(self) -> str:
        return "selfhosted_huggingface_pipeline"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        return self.client(
            pipeline=self.pipeline_ref, prompt=prompt, stop=stop, **kwargs
        )