Spaces:
Running
Running
File size: 1,842 Bytes
06bf5a5 c997974 06bf5a5 c997974 b1ab347 579a105 06bf5a5 c997974 b3ec1fd c997974 06bf5a5 c997974 06bf5a5 c997974 579a105 c997974 b3ec1fd c997974 579a105 c997974 06bf5a5 c997974 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
from huggingface_hub import InferenceClient
from auditqa.process_chunks import getconfig
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain_community.llms import HuggingFaceEndpoint
from langchain_community.chat_models.huggingface import ChatHuggingFace
import os
from dotenv import load_dotenv
load_dotenv()
# TESTING DEBUG LOG
from auditqa.logging_config import setup_logging
setup_logging()
import logging
logger = logging.getLogger(__name__)
model_config = getconfig("model_params.cfg")
# NVIDIA_SERVER = os.environ["NVIDIA_SERVERLESS"] #TESTING
HF_token = os.environ["LLAMA_3_1"]
def nvidia_client():
logger.info("NVIDIA client activated")
""" returns the nvidia server client """
try:
NVIDIA_SERVER = os.environ["NVIDIA_SERVERLESS"]
client = InferenceClient(
base_url=model_config.get('reader','NVIDIA_ENDPOINT'),
api_key=NVIDIA_SERVER)
print("getting nvidia client")
return client
except KeyError:
raise KeyError("NVIDIA_SERVERLESS environment variable not set. Required for NVIDIA endpoint.")
# TESTING VERSION
def dedicated_endpoint():
logger.info("Serverless endpoint activated")
try:
HF_token = os.environ["LLAMA_3_1"]
if not HF_token:
raise ValueError("LLAMA_3_1 environment variable is empty")
model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
logger.info(f"Initializing InferenceClient with model: {model_id}")
client = InferenceClient(
model=model_id,
api_key=HF_token,
)
logger.info("Serverless InferenceClient initialization successful")
return client
except Exception as e:
logger.error(f"Error initializing dedicated endpoint: {str(e)}")
raise
|