lfg

app.py

@@ -78,9 +78,11 @@ def process(
     # import pdb; pdb.set_trace()
 
     ocr_bbox_rslt, is_goal_filtered = check_ocr_box(image_save_path, display_img = False, output_bb_format='xyxy', goal_filtering=None, easyocr_args={'paragraph': False, 'text_threshold':0.9}, use_paddleocr=True)
+    
+
     text, ocr_bbox = ocr_bbox_rslt
     # print('prompt:', prompt)
-    dino_labled_img, label_coordinates, parsed_content_list = get_som_labeled_img(image_save_path, yolo_model, BOX_TRESHOLD = box_threshold, output_coord_in_ratio=True, ocr_bbox=ocr_bbox,draw_bbox_config=draw_bbox_config, caption_model_processor=caption_model_processor, ocr_text=text,iou_threshold=iou_threshold)
+    dino_labled_img, label_coordinates, parsed_content_list = get_som_labeled_img(image_save_path, yolo_model, BOX_TRESHOLD = box_threshold, output_coord_in_ratio=False, ocr_bbox=ocr_bbox,draw_bbox_config=draw_bbox_config, caption_model_processor=caption_model_processor, ocr_text=text,iou_threshold=iou_threshold)
     image = Image.open(io.BytesIO(base64.b64decode(dino_labled_img)))
     print('finish processing')
     parsed_content_list = '\n'.join(parsed_content_list)
@@ -93,7 +95,22 @@ def process(
     #     coords_formatted = [f"{coord:.1f}" for coord in coords_list]
     #     coordinates_text += f"Box {box_id}: [{coords_formatted[0]}, {coords_formatted[1]}, {coords_formatted[2]}, {coords_formatted[3]}]\n"
         
-    return image, str(parsed_content_list), str(label_coordinates)
+    combined_content = []
+    for i, content in enumerate(parsed_content_list):
+        if content.startswith('Text Box ID'):
+            box_id = str(i)
+        else:
+            # Extract the ID number from Icon Box ID format
+            box_id = content.split('Icon Box ID ')[1].split(':')[0]
+        
+        coords = label_coordinates.get(box_id)
+        if coords is not None:  # Changed from 'if coords:' to handle numpy arrays
+            coords_str = [round(x) for x in coords]  # Convert numpy values to rounded integers
+            combined_content.append(f"{content} | Coordinates: {coords_str}")
+        else:
+            combined_content.append(content)
+
+    return image, str(parsed_content_list), str(combined_content)
 
 
 with gr.Blocks() as demo: