param async_client: Any = None¶
param cache: Optional[bool] = None¶
param callback_manager: Optional[BaseCallbackManager] = None¶
param callbacks: Callbacks = None¶
param client: Any = None¶
param inference_server_url: str = ''¶
param max_new_tokens: int = 512¶
param metadata: Optional[Dict[str, Any]] = None¶
Metadata to add to the run trace.
param repetition_penalty: Optional[float] = None¶
param seed: Optional[int] = None¶
param server_kwargs: Dict[str, Any] [Optional]¶
param stop_sequences: List[str] [Optional]¶
param streaming: bool = False¶
param tags: Optional[List[str]] = None¶
Tags to add to the run trace.
param temperature: float = 0.8¶
param timeout: int = 120¶
param top_k: Optional[int] = None¶
param top_p: Optional[float] = 0.95¶
param truncate: Optional[int] = None¶
param typical_p: Optional[float] = 0.95¶
param verbose: bool [Optional]¶
Whether to print out response text.
__call__(prompt: str, stop: Optional[List[str]] = None, callbacks: Optional[Union[List[BaseCallbackHandler], BaseCallbackManager]] = None, *, tags: Optional[List[str]] = None, metadata: Optional[Dict[str, Any]] = None, **kwargs: Any) → str¶
Check Cache and run the LLM on the given prompt and input.
async abatch(inputs: List[Union[PromptValue, str, List[BaseMessage]]], config: Optional[Union[RunnableConfig, List[RunnableConfig]]] = None, max_concurrency: Optional[int] = None, **kwargs: Any) → List[str]¶