import time
import json
import requests
import gradio as gr
STYLE = """
.no-border {
border: none !important;
}
.group-border {
padding: 10px;
border-width: 1px;
border-radius: 10px;
border-color: gray;
border-style: solid;
box-shadow: 1px 1px 3px;
}
.control-label-font {
font-size: 13pt !important;
}
.control-button {
background: none !important;
border-color: #69ade2 !important;
border-width: 2px !important;
color: #69ade2 !important;
}
.center {
text-align: center;
}
.right {
text-align: right;
}
.no-label {
padding: 0px !important;
}
.no-label > label > span {
display: none;
}
.small-big {
font-size: 12pt !important;
}
"""
def avaliable_providers():
providers = []
headers = {
"Content-Type": "application/json",
}
endpoint_url = "https://api.endpoints.huggingface.cloud/v2/provider"
response = requests.get(endpoint_url, headers=headers)
providers = {}
for provider in response.json()['vendors']:
if provider['status'] == 'available':
regions = {}
availability = False
for region in provider['regions']:
if region["status"] == "available":
regions[region['name']] = {
"label": region['label'],
"computes": region['computes']
}
availability = True
if availability:
providers[provider['name']] = regions
return providers
providers = avaliable_providers()
def update_regions(provider):
avalialbe_regions = []
regions = providers[provider]
for region, attributes in regions.items():
avalialbe_regions.append(f"{region}[{attributes['label']}]")
return gr.Dropdown.update(
choices=avalialbe_regions,
value=avalialbe_regions[0] if len(avalialbe_regions) > 0 else None
)
def update_compute_options(provider, region):
avalialbe_compute_options = []
computes = providers[provider][region.split("[")[0].strip()]["computes"]
for compute in computes:
if compute['status'] == 'available':
accelerator = compute['accelerator']
numAccelerators = compute['numAccelerators']
memoryGb = compute['memoryGb']
architecture = compute['architecture']
instanceType = compute['instanceType']
pricePerHour = compute['pricePerHour']
type = f"{numAccelerators}vCPU {memoryGb} · {architecture}" if accelerator == "cpu" else f"{numAccelerators}x {architecture}"
avalialbe_compute_options.append(
f"{compute['accelerator'].upper()} [{compute['instanceSize']}] · {type} · {instanceType} · ${pricePerHour}/hour"
)
return gr.Dropdown.update(
choices=avalialbe_compute_options,
value=avalialbe_compute_options[0] if len(avalialbe_compute_options) > 0 else None
)
def submit(
hf_account_input,
hf_token_input,
endpoint_name_input,
provider_selector,
region_selector,
repository_selector,
task_selector,
framework_selector,
compute_selector,
min_node_selector,
max_node_selector,
security_selector,
custom_kernel,
max_input_length,
max_tokens,
max_batch_prefill_token,
max_batch_total_token
):
compute_resources = compute_selector.split("·")
accelerator = compute_resources[0][:3].strip()
size_l_index = compute_resources[0].index("[") - 1
size_r_index = compute_resources[0].index("]")
size = compute_resources[0][size_l_index : size_r_index].strip()
type = compute_resources[-2].strip()
payload = {
"accountId": hf_account_input.strip(),
"compute": {
"accelerator": accelerator.lower(),
"instanceSize": size[1:],
"instanceType": type,
"scaling": {
"maxReplica": int(max_node_selector),
"minReplica": int(min_node_selector)
}
},
"model": {
"framework": framework_selector.lower(),
"image": {
"custom": {
"health_route": "/health",
"env": {
"DISABLE_CUSTOM_KERNELS": "true" if custom_kernel == "Enabled" else "false",
"MAX_BATCH_PREFILL_TOKENS": str(max_batch_prefill_token),
"MAX_BATCH_TOTAL_TOKENS": str(max_batch_total_token),
"MAX_INPUT_LENGTH": str(max_input_length),
"MAX_TOTAL_TOKENS": str(max_tokens),
"MODEL_ID": repository_selector.lower(),
# QUANTIZE: 'bitsandbytes' | 'gptq';
},
"url": "ghcr.io/huggingface/text-generation-inference:1.0.1",
}
},
"repository": repository_selector.lower(),
# "revision": "main",
"task": task_selector.lower()
},
"name": endpoint_name_input.strip().lower(),
"provider": {
"region": region_selector.split("[")[0].lower(),
"vendor": provider_selector.lower()
},
"type": security_selector.lower()
}
print(payload)
payload = json.dumps(payload)
print(payload)
headers = {
"Authorization": f"Bearer {hf_token_input.strip()}",
"Content-Type": "application/json",
}
endpoint_url = f"https://api.endpoints.huggingface.cloud/v2/endpoint/"#{hf_account_input.strip()}"
print(endpoint_url)
response = requests.post(endpoint_url, headers=headers, data=payload)
if response.status_code == 400:
return f"{response.text}. Malformed data in {payload}"
elif response.status_code == 401:
return "Invalid token"
elif response.status_code == 409:
return f"Endpoint {endpoint_name_input} already exists"
elif response.status_code == 202:
return f"Endpoint {endpoint_name_input} created successfully on {provider_selector.lower()} using {repository_selector.lower()}@main.\nPlease check out the progress at https://ui.endpoints.huggingface.co/endpoints."
else:
return f"something went wrong {response.status_code} = {response.text}"
with gr.Blocks(css=STYLE) as hf_endpoint:
with gr.Tab("Hugging Face", elem_classes=["no-border"]):
gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
with gr.Column(elem_classes=["group-border"]):
with gr.Row():
with gr.Column():
gr.Markdown("""### Hugging Face account ID (name)""")
hf_account_input = gr.Textbox(show_label=False, elem_classes=["no-label", "small-big"])
with gr.Column():
gr.Markdown("### Hugging Face access token")
hf_token_input = gr.Textbox(show_label=False, type="password", elem_classes=["no-label", "small-big"])
with gr.Row():
with gr.Column():
gr.Markdown("""### Target model
Model from the Hugging Face hub""")
repository_selector = gr.Textbox(
value="NousResearch/Nous-Hermes-Llama2-13b",
interactive=False,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Target model version(branch)
Branch name of the Model""")
revision_selector = gr.Textbox(
value=f"main",
interactive=False,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column(elem_classes=["group-border"]):
with gr.Column():
gr.Markdown("""### Endpoint name
Name for your new endpoint""")
endpoint_name_input = gr.Textbox(show_label=False, elem_classes=["no-label", "small-big"])
with gr.Row():
with gr.Column():
gr.Markdown("""### Cloud Provider""")
provider_selector = gr.Dropdown(
choices=providers.keys(),
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Cloud Region""")
region_selector = gr.Dropdown(
[],
value="",
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Row(visible=False):
with gr.Column():
gr.Markdown("### Task")
task_selector = gr.Textbox(
value="text-generation",
interactive=False,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("### Framework")
framework_selector = gr.Textbox(
value="PyTorch",
interactive=False,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Compute Instance Type""")
compute_selector = gr.Dropdown(
[],
value="",
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Row():
with gr.Column():
gr.Markdown("""### Min Number of Nodes""")
min_node_selector = gr.Number(
value=1,
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Max Number of Nodes""")
max_node_selector = gr.Number(
value=1,
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Security Level""")
security_selector = gr.Radio(
choices=["Protected", "Public", "Private"],
value="Public",
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column(elem_classes=["group-border"]):
with gr.Accordion("Serving Container", open=False, elem_classes=["no-border"]):
with gr.Column():
gr.Markdown("""### Container Type
Text Generation Inference is an optimized container for text generation task""")
_ = gr.Textbox("Text Generation Inference", show_label=False, elem_classes=["no-label", "small-big"])
with gr.Row():
with gr.Column():
gr.Markdown("""### Custom Cuda Kernels
TGI uses custom kernels to speed up inference for some models. You can try disabling them if you encounter issues.""")
custom_kernel = gr.Dropdown(
value="Enabled",
choices=["Enabled", "Disabled"],
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Quantization
Quantization can reduce the model size and improve latency, with little degradation in model accuracy.""")
_ = gr.Dropdown(
value="None",
choices=["None", "Bitsandbytes", "GPTQ"],
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Row():
with gr.Column():
gr.Markdown("""### Max Input Length (per Query)
Increasing this value can impact the amount of RAM required. Some models can only handle a finite range of sequences.""")
max_input_length = gr.Number(
value=1024,
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Max Number of Tokens (per Query)
The larger this value, the more memory each request will consume and the less effective batching can be.""")
max_tokens = gr.Number(
value=1512,
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Row():
with gr.Column():
gr.Markdown("""### Max Batch Prefill Tokens
Number of prefill tokens used during continuous batching. It can be useful to adjust this number since the prefill operation is memory-intensive and compute-bound.""")
max_batch_prefill_token = gr.Number(
value=2048,
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
with gr.Column():
gr.Markdown("""### Max Batch Total Tokens
Number of tokens that can be passed before forcing waiting queries to be put on the batch. A value of 1000 can fit 10 queries of 100 tokens or a single query of 1000 tokens.""")
max_batch_total_token = gr.Number(
value=None,
interactive=True,
show_label=False,
elem_classes=["no-label", "small-big"]
)
submit_button = gr.Button(
value="Submit",
elem_classes=["control-label-font", "control-button"]
)
status_txt = gr.Textbox(
value="any status update will be displayed here",
interactive=False,
elem_classes=["no-label"]
)
provider_selector.change(update_regions, inputs=provider_selector, outputs=region_selector)
region_selector.change(update_compute_options, inputs=[provider_selector, region_selector], outputs=compute_selector)
submit_button.click(
submit,
inputs=[
hf_account_input,
hf_token_input,
endpoint_name_input,
provider_selector,
region_selector,
repository_selector,
task_selector,
framework_selector,
compute_selector,
min_node_selector,
max_node_selector,
security_selector,
custom_kernel,
max_input_length,
max_tokens,
max_batch_prefill_token,
max_batch_total_token],
outputs=status_txt)
with gr.Tab("AWS", elem_classes=["no-border"]):
gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
with gr.Tab("GCP", elem_classes=["no-border"]):
gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
with gr.Tab("Azure", elem_classes=["no-border"]):
gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
with gr.Tab("Lambdalabs", elem_classes=["no-border"]):
gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
hf_endpoint.launch(enable_queue=True, debug=True)