added inference

app.py

@@ -36,6 +36,7 @@ import logging
 import uuid
 import json
 import os
+import requests
 
 app = Flask(__name__, static_folder='static')
 
@@ -122,6 +123,55 @@ def log():
         json.dump(existing_data, f)
     return jsonify({'message': 'Data added successfully', 'data': existing_data}), 201
 
+@app.route("/demo_inference", methods=['GET'])
+@cross_origin()
+def demo_inference():
+    args = request.args
+    # model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"
+    model_id = args.get('model_id', default="meta-llama/Llama-4-Scout-17B-16E-Instruct")
+    temperature = 0.5
+    max_new_tokens = 1000
+
+    hf_token, hf_url = get_credentials.get_credentials()
+
+    prompt = args.get('prompt')
+
+    API_URL = "https://router.huggingface.co/together/v1/chat/completions"
+    headers = {
+        "Authorization": f"Bearer {hf_token}",
+    }
+
+    response = requests.post(
+        API_URL,
+        headers=headers, 
+        json={
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": prompt
+                        },
+                    ]
+                }
+            ],
+            "model": model_id,
+            'temperature': temperature,
+            'max_new_tokens': max_new_tokens,
+        }
+    )
+    try:
+        response = response.json()["choices"][0]["message"]
+        response.update({
+            'model_id': model_id,
+            'temperature': temperature,
+            'max_new_tokens': max_new_tokens,
+        })
+        return response
+    except:
+        return response.text, response.status_code
+
 if __name__=='__main__':
     debug_mode = os.getenv('FLASK_DEBUG', 'True').lower() in ['true', '1', 't']
     app.run(host='0.0.0.0', port='7860', debug=debug_mode)

static/demo/index.html

@@ -7,6 +7,8 @@
     <link rel="stylesheet" href="https://unpkg.com/carbon-components/css/carbon-components.min.css">
     <script type="text/javascript" src="static/demo/js/d3.v7.min.js"></script>
     <script type="text/javascript" src="static/demo/js/jquery-3.7.1.min.js"></script>
+    <!-- <script type="text/javascript" src="js/d3.v7.min.js"></script>
+    <script type="text/javascript" src="js/jquery-3.7.1.min.js"></script> -->
     <style type="text/css">
         div.tooltip {
             position: absolute;
@@ -390,22 +392,6 @@
 
         // Generation request
         $( "#demo" ).on( "submit", function(e){ // Hugging Face
-
-            var out = "" ;
-            const ajax_headers = {
-                'Content-Type': 'application/json',
-                'Accept': 'application/json',
-                'Access-Control-Allow-Headers': '*',
-                'Authorization' : 'Bearer <include-token-here>',
-            };
-            $.ajaxSetup({
-                headers: ajax_headers
-            });
-
-            if( ajax_headers['Authorization'] == "Bearer <include-token-here>" ){
-                console.error( "Please inform your authorization token in the ajax header setup." ) ;
-            }
-            else{
                 $( "#generate" ).toggleClass( "bx--btn--disabled" ) ;
                 ( function loading_animation(){
                     if( $( "#outcome" ).attr( "placeholder" ) == "" ){
@@ -420,21 +406,13 @@
                     setTimeout( loading_animation, 500 );
                 } )()
 
-                var temperature = 0.5 ;
-                var max_new_tokens = 1000 ;
-                var model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"
                 $.ajax({
-                    type: "POST",
-                    url: "https://api-inference.huggingface.co/models/" + model_id,
-                    data: JSON.stringify({
-                                    "inputs":  $("#prompt").val(),
-                                    "parameters": {
-                                        "temperature": temperature,
-                                        "max_new_tokens": max_new_tokens
-                                    }
-                                }),
-                    crossDomain: true,
+                    url: encodeURI("/demo_inference?prompt=" + $("#prompt").val()),
+                    dataType: 'json',
                     success: function(data){
+                        
+                        console.log("Inference response")
+                        console.log(data)
                         // Resetting the status of the button
                         $( "#generate" ).toggleClass( "bx--btn--disabled" ) ;
 
@@ -444,7 +422,10 @@
                             $( "#demo" ).data( "timeoutId", "" ) ;
                         }
 
-                        out = data[0].generated_text.split("");
+                        out = data.content.split("");
+                        model_id = data.model_id;
+                        temperature = data.temperature
+                        max_new_tokens = data.max_new_tokens
 
                         $( "#outcome" ).append( "\n\n+ ------------------------------------\n| Model: " + model_id + "\n| Temperature: " + temperature + "\n| Max new tokens: " + max_new_tokens + "\n+ ------------------------------------\n\n" ) ;
                         // Animating the generated output
@@ -456,9 +437,13 @@
                                 $( "#demo" ).data( "timeoutId", timeoutId ) ;
                             }
                         } )()
+                    },
+                    error: function(data) {
+                        out = data.responseJSON.error.message
+                        $( "#outcome" ).val(out);
                     }
-                });
-            }
+                })
+
             // Returning false so the form keeps user in the same page
             return false;
         });