Try to handle inputs slightly different

app.py

@@ -36,12 +36,13 @@ def translate(python, progress=gr.Progress()):
                         messages_for(python),
                         tokenize=False,
                         add_generation_prompt=True,
-                        return_tensors="pt").to(model.device)
-    inputs = tokenizer(formatted_prompt, return_tensors="pt", padding=True)
+                        return_tensors="pt")
+    inputs = tokenizer(formatted_prompt, return_tensors="pt", padding=True).to(model.device)
     attention_mask = inputs["attention_mask"]
+    input_ids = inputs['input_ids']
 
     outputs = model.generate(
-        inputs['input_ids'],
+        input_ids,
         attention_mask=attention_mask,
         max_new_tokens=1024,
         do_sample=False,
@@ -49,7 +50,7 @@ def translate(python, progress=gr.Progress()):
         eos_token_id=tokenizer.eos_token_id,
     )
     progress(1, desc="Finished")
-    return tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)
+    return tokenizer.decode(outputs[0][len(input_ids[0]):], skip_special_tokens=True)
     '''
     generation_kwargs = dict(
                             inputs,